metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ceste...@apache.org
Subject [3/9] incubator-metron git commit: METRON-56 Create unified enrichment topology (merrimanr via cestella) closes apache/incubator-metron#33
Date Tue, 01 Mar 2016 19:29:58 GMT
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/sourcefire/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/sourcefire/remote.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/sourcefire/remote.yaml
index 957677b..5bc5f76 100644
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/sourcefire/remote.yaml
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/sourcefire/remote.yaml
@@ -21,143 +21,10 @@ config:
 components:
     -   id: "parser"
         className: "org.apache.metron.parsing.parsers.BasicSourcefireParser"
-    -   id: "jdbcConfig"
-        className: "org.apache.metron.enrichment.adapters.jdbc.MySqlConfig"
-        properties:
-            -   name: "host"
-                value: "${mysql.ip}"
-            -   name: "port"
-                value: ${mysql.port}
-            -   name: "username"
-                value: "${mysql.username}"
-            -   name: "password"
-                value: "${mysql.password}"
-            -   name: "table"
-                value: "GEO"
-    -   id: "geoEnrichmentAdapter"
-        className: "org.apache.metron.enrichment.adapters.geo.GeoAdapter"
-        configMethods:
-            -   name: "withJdbcConfig"
-                args:
-                    - ref: "jdbcConfig"
-    -   id: "geoEnrichment"
-        className: "org.apache.metron.domain.Enrichment"
-        properties:
-            -   name: "name"
-                value:  "geo"
-            -   name: "fields"
-                value: ["ip_src_addr", "ip_dst_addr"]
-            -   name: "adapter"
-                ref: "geoEnrichmentAdapter"
-    -   id: "hostEnrichmentAdapter"
-        className: "org.apache.metron.enrichment.adapters.host.HostFromJSONListAdapter"
+    -   id: "writer"
+        className: "org.apache.metron.writer.KafkaWriter"
         constructorArgs:
-            - '${org.apache.metron.enrichment.host.known_hosts}'
-    -   id: "hostEnrichment"
-        className: "org.apache.metron.domain.Enrichment"
-        properties:
-            -   name: "name"
-                value:  "host"
-            -   name: "fields"
-                value: ["ip_src_addr", "ip_dst_addr"]
-            -   name: "adapter"
-                ref: "hostEnrichmentAdapter"
-    -   id: "enrichments"
-        className: "java.util.ArrayList"
-        configMethods:
-            -   name: "add"
-                args:
-                    - ref: "geoEnrichment"
-            -   name: "add"
-                args:
-                    - ref: "hostEnrichment"
-    -   id: "indexAdapter"
-        className: "org.apache.metron.indexing.adapters.ESTimedRotatingAdapter"
-    -   id: "alertsConfig"
-        className: "java.util.HashMap"
-        configMethods:
-            -   name: "put"
-                args: ["whitelist_table_name", "ip_whitelist"]
-            -   name: "put"
-                args: ["blacklist_table_name", "ip_blacklist"]
-            -   name: "put"
-                args: ["quorum", "mon.cluster2.ctolab.hortonworks.com, nn1.cluster2.ctolab.hortonworks.com, nn2.cluster2.ctolab.hortonworks.com"]
-            -   name: "put"
-                args: ["port", "2181"]
-            -   name: "put"
-                args: ["_MAX_CACHE_SIZE_OBJECTS_NUM", "3600"]
-            -   name: "put"
-                args: ["_MAX_TIME_RETAIN_MINUTES", "1000"]
-    -   id: "alertsAdapter"
-        className: "org.apache.metron.alerts.adapters.CIFAlertsAdapter"
-        constructorArgs:
-            - ref: "alertsConfig"
-    -   id: "alertsIdentifier"
-        className: "org.json.simple.JSONObject"
-        configMethods:
-            -   name: "put"
-                args: ["environment", "local"]
-            -   name: "put"
-                args: ["topology", "sourcefire"]
-    -   id: "metricConfig"
-        className: "org.apache.commons.configuration.BaseConfiguration"
-        configMethods:
-            -   name: "setProperty"
-                args:
-                    - "org.apache.metron.metrics.reporter.graphite"
-                    - "${org.apache.metron.metrics.reporter.graphite}"
-            -   name: "setProperty"
-                args:
-                    - "org.apache.metron.metrics.reporter.console"
-                    - "${org.apache.metron.metrics.reporter.console}"
-            -   name: "setProperty"
-                args:
-                    - "org.apache.metron.metrics.reporter.jmx"
-                    - "${org.apache.metron.metrics.reporter.jmx}"
-            -   name: "setProperty"
-                args:
-                    - "org.apache.metron.metrics.graphite.address"
-                    - "${org.apache.metron.metrics.graphite.address}"
-            -   name: "setProperty"
-                args:
-                    - "org.apache.metron.metrics.graphite.port"
-                    - "${org.apache.metron.metrics.graphite.port}"
-            -   name: "setProperty"
-                args:
-                    - "org.apache.metron.metrics.TelemetryParserBolt.acks"
-                    - "${org.apache.metron.metrics.TelemetryParserBolt.acks}"
-            -   name: "setProperty"
-                args:
-                    - "org.apache.metron.metrics.TelemetryParserBolt.emits"
-                    - "${org.apache.metron.metrics.TelemetryParserBolt.emits}"
-            -   name: "setProperty"
-                args:
-                    - "org.apache.metron.metrics.TelemetryParserBolt.fails"
-                    - "${org.apache.metron.metrics.TelemetryParserBolt.fails}"
-            -   name: "setProperty"
-                args:
-                    - "org.apache.metron.metrics.GenericEnrichmentBolt.acks"
-                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.acks}"
-            -   name: "setProperty"
-                args:
-                    - "org.apache.metron.metrics.GenericEnrichmentBolt.emits"
-                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.emits}"
-            -   name: "setProperty"
-                args:
-                    - "org.apache.metron.metrics.GenericEnrichmentBolt.fails"
-                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.fails}"
-            -   name: "setProperty"
-                args:
-                    - "org.apache.metron.metrics.TelemetryIndexingBolt.acks"
-                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.acks}"
-            -   name: "setProperty"
-                args:
-                    - "org.apache.metron.metrics.TelemetryIndexingBolt.emits"
-                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.emits}"
-            -   name: "setProperty"
-                args:
-                    - "org.apache.metron.metrics.TelemetryIndexingBolt.fails"
-                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.fails}"
+            - "${kafka.broker}"
     -   id: "zkHosts"
         className: "storm.kafka.ZkHosts"
         constructorArgs:
@@ -168,18 +35,28 @@ components:
             # zookeeper hosts
             - ref: "zkHosts"
             # topic name
-            - "${spout.kafka.topic.pcap}"
+            - "${spout.kafka.topic.snort}"
             # zk root
             - ""
             # id
-            - "${spout.kafka.topic.pcap}"
+            - "${spout.kafka.topic.snort}"
         properties:
-            -   name: "forceFromStart"
+            -   name: "ignoreZkOffsets"
                 value: true
             -   name: "startOffsetTime"
                 value: -1
 
 spouts:
+    -   id: "testingSpout"
+        className: "org.apache.metron.test.spouts.GenericInternalTestSpout"
+        parallelism: 1
+        configMethods:
+            -   name: "withFilename"
+                args:
+                    - "SampleInput/YafExampleOutput"
+            -   name: "withRepeating"
+                args:
+                    - true
     -   id: "kafkaSpout"
         className: "storm.kafka.KafkaSpout"
         constructorArgs:
@@ -187,229 +64,16 @@ spouts:
 
 bolts:
     -   id: "parserBolt"
-        className: "org.apache.metron.bolt.TelemetryParserBolt"
-        configMethods:
-            -   name: "withMessageParser"
-                args:
-                    - ref: "parser"
-            -   name: "withEnrichments"
-                args:
-                    - ref: "enrichments"
-    -   id: "indexingBolt"
-        className: "org.apache.metron.indexing.TelemetryIndexingBolt"
-        configMethods:
-            -   name: "withIndexIP"
-                args:
-                    - "${es.ip}"
-            -   name: "withIndexPort"
-                args:
-                    - ${es.port}
-            -   name: "withClusterName"
-                args:
-                    - "${es.clustername}"
-            -   name: "withIndexName"
-                args:
-                    - "sourcefire_index"
-            -   name: "withIndexTimestamp"
-                args:
-                    - "yyyy.MM.dd.hh"
-            -   name: "withDocumentName"
-                args:
-                    - "sourcefire_doc"
-            -   name: "withBulk"
-                args:
-                    - 1
-            -   name: "withIndexAdapter"
-                args:
-                    - ref: "indexAdapter"
-            -   name: "withMetricConfiguration"
-                args:
-                    - ref: "metricConfig"
-    -   id: "alertsBolt"
-        className: "org.apache.metron.alerts.TelemetryAlertsBolt"
-        configMethods:
-            -   name: "withIdentifier"
-                args:
-                    - ref: "alertsIdentifier"
-            -   name: "withMaxCacheSize"
-                args: [1000]
-            -   name: "withMaxTimeRetain"
-                args: [3600]
-            -   name: "withAlertsAdapter"
-                args:
-                    - ref: "alertsAdapter"
-            -   name: "withOutputFieldName"
-                args: ["message"]
-            -   name: "withMetricConfiguration"
-                args:
-                    - ref: "metricConfig"
-    -   id: "alertsIndexingBolt"
-        className: "org.apache.metron.indexing.TelemetryIndexingBolt"
-        configMethods:
-            -   name: "withIndexIP"
-                args:
-                    - "${es.ip}"
-            -   name: "withIndexPort"
-                args:
-                    - ${es.port}
-            -   name: "withClusterName"
-                args:
-                    - "${es.clustername}"
-            -   name: "withIndexName"
-                args:
-                    - "alert"
-            -   name: "withIndexTimestamp"
-                args:
-                    - "yyyy.MM.ww"
-            -   name: "withDocumentName"
-                args:
-                    - "sourcefire_alert"
-            -   name: "withBulk"
-                args:
-                    - 1
-            -   name: "withIndexAdapter"
-                args:
-                    - ref: "indexAdapter"
-            -   name: "withMetricConfiguration"
-                args:
-                    - ref: "metricConfig"
-    -   id: "errorIndexingBolt"
-        className: "org.apache.metron.indexing.TelemetryIndexingBolt"
-        configMethods:
-            -   name: "withIndexIP"
-                args:
-                    - "${es.ip}"
-            -   name: "withIndexPort"
-                args:
-                    - ${es.port}
-            -   name: "withClusterName"
-                args:
-                    - "${es.clustername}"
-            -   name: "withIndexName"
-                args:
-                    - "error"
-            -   name: "withIndexTimestamp"
-                args:
-                    - "yyyy.MM"
-            -   name: "withDocumentName"
-                args:
-                    - "sourcefire_error"
-            -   name: "withBulk"
-                args:
-                    - 1
-            -   name: "withIndexAdapter"
-                args:
-                    - ref: "indexAdapter"
-            -   name: "withMetricConfiguration"
-                args:
-                    - ref: "metricConfig"
-    -   id: "geoEnrichmentBolt"
-        className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt"
-        configMethods:
-            -   name: "withEnrichment"
-                args:
-                    - ref: "geoEnrichment"
-            -   name: "withMaxCacheSize"
-                args: [10000]
-            -   name: "withMaxTimeRetain"
-                args: [10]
-    -   id: "hostEnrichmentBolt"
-        className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt"
-        configMethods:
-            -   name: "withEnrichment"
-                args:
-                    - ref: "hostEnrichment"
-            -   name: "withMaxCacheSize"
-                args: [10000]
-            -   name: "withMaxTimeRetain"
-                args: [10]
-    -   id: "joinBolt"
-        className: "org.apache.metron.enrichment.bolt.EnrichmentJoinBolt"
-        configMethods:
-        -   name: "withEnrichments"
-            args:
-                - ref: "enrichments"
-        -   name: "withMaxCacheSize"
-            args: [10000]
-        -   name: "withMaxTimeRetain"
-            args: [10]
+        className: "org.apache.metron.bolt.ParserBolt"
+        constructorArgs:
+            - "${kafka.zk}"
+            - "${spout.kafka.topic.snort}"
+            - ref: "parser"
+            - ref: "writer"
 
 streams:
-    -   name: "spout -> parser"
+    -   name: "spout -> bolt"
         from: "kafkaSpout"
         to: "parserBolt"
         grouping:
             type: SHUFFLE
-    -   name: "parser -> host"
-        from: "parserBolt"
-        to: "hostEnrichmentBolt"
-        grouping:
-            streamId: "host"
-            type: FIELDS
-            args: ["key"]
-    -   name: "parser -> geo"
-        from: "parserBolt"
-        to: "geoEnrichmentBolt"
-        grouping:
-            streamId: "geo"
-            type: FIELDS
-            args: ["key"]
-    -   name: "parser -> join"
-        from: "parserBolt"
-        to: "joinBolt"
-        grouping:
-            streamId: "message"
-            type: FIELDS
-            args: ["key"]
-    -   name: "geo -> join"
-        from: "geoEnrichmentBolt"
-        to: "joinBolt"
-        grouping:
-            streamId: "geo"
-            type: FIELDS
-            args: ["key"]
-    -   name: "host -> join"
-        from: "hostEnrichmentBolt"
-        to: "joinBolt"
-        grouping:
-            streamId: "host"
-            type: FIELDS
-            args: ["key"]
-    -   name: "join -> alerts"
-        from: "joinBolt"
-        to: "alertsBolt"
-        grouping:
-            streamId: "message"
-            type: FIELDS
-            args: ["key"]
-    -   name: "alerts -> alertsIndexing"
-        from: "alertsBolt"
-        to: "alertsIndexingBolt"
-        grouping:
-            streamId: "message"
-            type: SHUFFLE
-    -   name: "join -> indexing"
-        from: "joinBolt"
-        to: "indexingBolt"
-        grouping:
-            streamId: "message"
-            type: FIELDS
-            args: ["key"]
-    -   name: "parser -> errors"
-        from: "parserBolt"
-        to: "errorIndexingBolt"
-        grouping:
-            streamId: "error"
-            type: SHUFFLE
-    -   name: "indexing -> errors"
-        from: "indexingBolt"
-        to: "errorIndexingBolt"
-        grouping:
-            streamId: "error"
-            type: SHUFFLE
-    -   name: "alerts -> errors"
-        from: "alertsBolt"
-        to: "errorIndexingBolt"
-        grouping:
-            streamId: "error"
-            type: SHUFFLE

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/sourcefire/test.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/sourcefire/test.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/sourcefire/test.yaml
new file mode 100644
index 0000000..e9e583a
--- /dev/null
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/sourcefire/test.yaml
@@ -0,0 +1,79 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: "sourcefire-test"
+config:
+    topology.workers: 1
+
+components:
+    -   id: "parser"
+        className: "org.apache.metron.parsing.parsers.BasicSourcefireParser"
+    -   id: "writer"
+        className: "org.apache.metron.writer.KafkaWriter"
+        constructorArgs:
+            - "${kafka.broker}"
+    -   id: "zkHosts"
+        className: "storm.kafka.ZkHosts"
+        constructorArgs:
+            - "${kafka.zk}"
+    -   id: "kafkaConfig"
+        className: "storm.kafka.SpoutConfig"
+        constructorArgs:
+            # zookeeper hosts
+            - ref: "zkHosts"
+            # topic name
+            - "${spout.kafka.topic.snort}"
+            # zk root
+            - ""
+            # id
+            - "${spout.kafka.topic.snort}"
+        properties:
+            -   name: "ignoreZkOffsets"
+                value: true
+            -   name: "startOffsetTime"
+                value: -2
+
+spouts:
+    -   id: "testingSpout"
+        className: "org.apache.metron.test.spouts.GenericInternalTestSpout"
+        parallelism: 1
+        configMethods:
+            -   name: "withFilename"
+                args:
+                    - "SampleInput/YafExampleOutput"
+            -   name: "withRepeating"
+                args:
+                    - false
+    -   id: "kafkaSpout"
+        className: "storm.kafka.KafkaSpout"
+        constructorArgs:
+            - ref: "kafkaConfig"
+
+bolts:
+    -   id: "parserBolt"
+        className: "org.apache.metron.bolt.ParserBolt"
+        constructorArgs:
+            - "${kafka.zk}"
+            - "${spout.kafka.topic.snort}"
+            - ref: "parser"
+            - ref: "writer"
+
+streams:
+    -   name: "spout -> bolt"
+        from: "kafkaSpout"
+        to: "parserBolt"
+        grouping:
+            type: SHUFFLE

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/yaf/local.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/yaf/local.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/yaf/local.yaml
deleted file mode 100644
index cf026a2..0000000
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/yaf/local.yaml
+++ /dev/null
@@ -1,192 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: "yaf-local"
-config:
-    topology.workers: 1
-
-components:
-    -   id: "yafParser"
-        className: "org.apache.metron.parsing.parsers.BasicYafParser"
-    -   id: "genericMessageFilter"
-        className: "org.apache.metron.filters.GenericMessageFilter"
-    -   id: "indexAdapter"
-        className: "org.apache.metron.indexing.adapters.ESTimedRotatingAdapter"
-    -   id: "metricConfig"
-        className: "org.apache.commons.configuration.BaseConfiguration"
-        configMethods:
-            -   name: "setProperty"
-                args:
-                    - "org.apache.metron.metrics.reporter.graphite"
-                    - "${org.apache.metron.metrics.reporter.graphite}"
-            -   name: "setProperty"
-                args:
-                    - "org.apache.metron.metrics.reporter.console"
-                    - "${org.apache.metron.metrics.reporter.console}"
-            -   name: "setProperty"
-                args:
-                    - "org.apache.metron.metrics.reporter.jmx"
-                    - "${org.apache.metron.metrics.reporter.jmx}"
-            -   name: "setProperty"
-                args:
-                    - "org.apache.metron.metrics.graphite.address"
-                    - "${org.apache.metron.metrics.graphite.address}"
-            -   name: "setProperty"
-                args:
-                    - "org.apache.metron.metrics.graphite.port"
-                    - "${org.apache.metron.metrics.graphite.port}"
-            -   name: "setProperty"
-                args:
-                    - "org.apache.metron.metrics.TelemetryParserBolt.acks"
-                    - "${org.apache.metron.metrics.TelemetryParserBolt.acks}"
-            -   name: "setProperty"
-                args:
-                    - "org.apache.metron.metrics.TelemetryParserBolt.emits"
-                    - "${org.apache.metron.metrics.TelemetryParserBolt.emits}"
-            -   name: "setProperty"
-                args:
-                    - "org.apache.metron.metrics.TelemetryParserBolt.fails"
-                    - "${org.apache.metron.metrics.TelemetryParserBolt.fails}"
-            -   name: "setProperty"
-                args:
-                    - "org.apache.metron.metrics.GenericEnrichmentBolt.acks"
-                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.acks}"
-            -   name: "setProperty"
-                args:
-                    - "org.apache.metron.metrics.GenericEnrichmentBolt.emits"
-                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.emits}"
-            -   name: "setProperty"
-                args:
-                    - "org.apache.metron.metrics.GenericEnrichmentBolt.fails"
-                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.fails}"
-            -   name: "setProperty"
-                args:
-                    - "org.apache.metron.metrics.TelemetryIndexingBolt.acks"
-                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.acks}"
-            -   name: "setProperty"
-                args:
-                    - "org.apache.metron.metrics.TelemetryIndexingBolt.emits"
-                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.emits}"
-            -   name: "setProperty"
-                args:
-                    - "org.apache.metron.metrics.TelemetryIndexingBolt.fails"
-                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.fails}"
-
-spouts:
-    -   id: "testingSpout"
-        className: "org.apache.metron.test.spouts.GenericInternalTestSpout"
-        parallelism: 1
-        configMethods:
-            -   name: "withFilename"
-                args:
-                    - "SampleInput/YafExampleOutput"
-            -   name: "withRepeating"
-                args:
-                    - true
-
-bolts:
-    -   id: "parserBolt"
-        className: "org.apache.metron.bolt.TelemetryParserBolt"
-        configMethods:
-            -   name: "withMessageParser"
-                args:
-                    - ref: "yafParser"
-    -   id: "indexingBolt"
-        className: "org.apache.metron.indexing.TelemetryIndexingBolt"
-        configMethods:
-            -   name: "withIndexIP"
-                args:
-                    - "${es.ip}"
-            -   name: "withIndexPort"
-                args:
-                    - ${es.port}
-            -   name: "withClusterName"
-                args:
-                    - "${es.clustername}"
-            -   name: "withIndexName"
-                args:
-                    - "yaf_index"
-            -   name: "withIndexTimestamp"
-                args:
-                    - "yyyy.MM.dd.hh"
-            -   name: "withDocumentName"
-                args:
-                    - "yaf_doc"
-            -   name: "withBulk"
-                args:
-                    - 1
-            -   name: "withIndexAdapter"
-                args:
-                    - ref: "indexAdapter"
-            -   name: "withMetricConfiguration"
-                args:
-                    - ref: "metricConfig"
-    -   id: "errorIndexingBolt"
-        className: "org.apache.metron.indexing.TelemetryIndexingBolt"
-        configMethods:
-            -   name: "withIndexIP"
-                args:
-                    - "${es.ip}"
-            -   name: "withIndexPort"
-                args:
-                    - ${es.port}
-            -   name: "withClusterName"
-                args:
-                    - "${es.clustername}"
-            -   name: "withIndexName"
-                args:
-                    - "error"
-            -   name: "withIndexTimestamp"
-                args:
-                    - "yyyy.MM"
-            -   name: "withDocumentName"
-                args:
-                    - "yaf_error"
-            -   name: "withBulk"
-                args:
-                    - 1
-            -   name: "withIndexAdapter"
-                args:
-                    - ref: "indexAdapter"
-            -   name: "withMetricConfiguration"
-                args:
-                    - ref: "metricConfig"
-
-streams:
-    -   name: "spout -> parser"
-        from: "testingSpout"
-        to: "parserBolt"
-        grouping:
-            type: SHUFFLE
-    -   name: "parser -> indexing"
-        from: "parserBolt"
-        to: "indexingBolt"
-        grouping:
-            streamId: "message"
-            type: FIELDS
-            args: ["key"]
-    -   name: "parser -> errors"
-        from: "parserBolt"
-        to: "errorIndexingBolt"
-        grouping:
-            streamId: "error"
-            type: SHUFFLE
-    -   name: "indexing -> errors"
-        from: "indexingBolt"
-        to: "errorIndexingBolt"
-        grouping:
-            streamId: "error"
-            type: SHUFFLE

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/yaf/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/yaf/remote.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/yaf/remote.yaml
index 65cff0f..98395e9 100644
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/yaf/remote.yaml
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/yaf/remote.yaml
@@ -14,76 +14,31 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-name: "yaf"
+name: "yaf-test"
 config:
     topology.workers: 1
 
+
 components:
-    -   id: "yafParser"
-        className: "org.apache.metron.parsing.parsers.BasicYafParser"
-    -   id: "genericMessageFilter"
-        className: "org.apache.metron.filters.GenericMessageFilter"
-    -   id: "indexAdapter"
-        className: "org.apache.metron.indexing.adapters.ESTimedRotatingAdapter"
-    -   id: "metricConfig"
-        className: "org.apache.commons.configuration.BaseConfiguration"
+    -   id: "parser"
+        className: "org.apache.metron.parsing.parsers.GrokParser"
+        constructorArgs:
+            - "/patterns/yaf"
+            - "YAF_DELIMITED"
         configMethods:
-            -   name: "setProperty"
-                args:
-                    - "org.apache.metron.metrics.reporter.graphite"
-                    - "${org.apache.metron.metrics.reporter.graphite}"
-            -   name: "setProperty"
-                args:
-                    - "org.apache.metron.metrics.reporter.console"
-                    - "${org.apache.metron.metrics.reporter.console}"
-            -   name: "setProperty"
-                args:
-                    - "org.apache.metron.metrics.reporter.jmx"
-                    - "${org.apache.metron.metrics.reporter.jmx}"
-            -   name: "setProperty"
-                args:
-                    - "org.apache.metron.metrics.graphite.address"
-                    - "${org.apache.metron.metrics.graphite.address}"
-            -   name: "setProperty"
-                args:
-                    - "org.apache.metron.metrics.graphite.port"
-                    - "${org.apache.metron.metrics.graphite.port}"
-            -   name: "setProperty"
-                args:
-                    - "org.apache.metron.metrics.TelemetryParserBolt.acks"
-                    - "${org.apache.metron.metrics.TelemetryParserBolt.acks}"
-            -   name: "setProperty"
-                args:
-                    - "org.apache.metron.metrics.TelemetryParserBolt.emits"
-                    - "${org.apache.metron.metrics.TelemetryParserBolt.emits}"
-            -   name: "setProperty"
-                args:
-                    - "org.apache.metron.metrics.TelemetryParserBolt.fails"
-                    - "${org.apache.metron.metrics.TelemetryParserBolt.fails}"
-            -   name: "setProperty"
-                args:
-                    - "org.apache.metron.metrics.GenericEnrichmentBolt.acks"
-                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.acks}"
-            -   name: "setProperty"
-                args:
-                    - "org.apache.metron.metrics.GenericEnrichmentBolt.emits"
-                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.emits}"
-            -   name: "setProperty"
+            -   name: "withTimestampField"
                 args:
-                    - "org.apache.metron.metrics.GenericEnrichmentBolt.fails"
-                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.fails}"
-            -   name: "setProperty"
+                    - "start_time"
+            -   name: "withTimeFields"
                 args:
-                    - "org.apache.metron.metrics.TelemetryIndexingBolt.acks"
-                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.acks}"
-            -   name: "setProperty"
+                    - ["start_time", "end_time"]
+            -   name: "withDateFormat"
                 args:
-                    - "org.apache.metron.metrics.TelemetryIndexingBolt.emits"
-                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.emits}"
-            -   name: "setProperty"
-                args:
-                    - "org.apache.metron.metrics.TelemetryIndexingBolt.fails"
-                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.fails}"
+                    - "yyyy-MM-dd HH:mm:ss.S"
+    -   id: "writer"
+        className: "org.apache.metron.writer.KafkaWriter"
+        constructorArgs:
+            - "${kafka.broker}"
     -   id: "zkHosts"
         className: "storm.kafka.ZkHosts"
         constructorArgs:
@@ -100,12 +55,24 @@ components:
             # id
             - "${spout.kafka.topic.yaf}"
         properties:
-            -   name: "forceFromStart"
+            -   name: "ignoreZkOffsets"
                 value: true
             -   name: "startOffsetTime"
                 value: -1
+            -   name: "socketTimeoutMs"
+                value: 1000000
 
 spouts:
+    -   id: "testingSpout"
+        className: "org.apache.metron.test.spouts.GenericInternalTestSpout"
+        parallelism: 1
+        configMethods:
+            -   name: "withFilename"
+                args:
+                    - "SampleInput/YafExampleOutput"
+            -   name: "withRepeating"
+                args:
+                    - true
     -   id: "kafkaSpout"
         className: "storm.kafka.KafkaSpout"
         constructorArgs:
@@ -113,94 +80,16 @@ spouts:
 
 bolts:
     -   id: "parserBolt"
-        className: "org.apache.metron.bolt.TelemetryParserBolt"
-        configMethods:
-            -   name: "withMessageParser"
-                args:
-                    - ref: "yafParser"
-    -   id: "indexingBolt"
-        className: "org.apache.metron.indexing.TelemetryIndexingBolt"
-        configMethods:
-            -   name: "withIndexIP"
-                args:
-                    - "${es.ip}"
-            -   name: "withIndexPort"
-                args:
-                    - ${es.port}
-            -   name: "withClusterName"
-                args:
-                    - "${es.clustername}"
-            -   name: "withIndexName"
-                args:
-                    - "yaf_index"
-            -   name: "withIndexTimestamp"
-                args:
-                    - "yyyy.MM.dd.hh"
-            -   name: "withDocumentName"
-                args:
-                    - "yaf_doc"
-            -   name: "withBulk"
-                args:
-                    - 1
-            -   name: "withIndexAdapter"
-                args:
-                    - ref: "indexAdapter"
-            -   name: "withMetricConfiguration"
-                args:
-                    - ref: "metricConfig"
-    -   id: "errorIndexingBolt"
-        className: "org.apache.metron.indexing.TelemetryIndexingBolt"
-        configMethods:
-            -   name: "withIndexIP"
-                args:
-                    - "${es.ip}"
-            -   name: "withIndexPort"
-                args:
-                    - ${es.port}
-            -   name: "withClusterName"
-                args:
-                    - "${es.clustername}"
-            -   name: "withIndexName"
-                args:
-                    - "error"
-            -   name: "withIndexTimestamp"
-                args:
-                    - "yyyy.MM"
-            -   name: "withDocumentName"
-                args:
-                    - "yaf_error"
-            -   name: "withBulk"
-                args:
-                    - 1
-            -   name: "withIndexAdapter"
-                args:
-                    - ref: "indexAdapter"
-            -   name: "withMetricConfiguration"
-                args:
-                    - ref: "metricConfig"
+        className: "org.apache.metron.bolt.ParserBolt"
+        constructorArgs:
+            - "${kafka.zk}"
+            - "yaf"
+            - ref: "parser"
+            - ref: "writer"
 
 streams:
-    -   name: "spout -> parser"
+    -   name: "spout -> bolt"
         from: "kafkaSpout"
         to: "parserBolt"
         grouping:
             type: SHUFFLE
-    -   name: "parser -> indexing"
-        from: "parserBolt"
-        to: "indexingBolt"
-        grouping:
-            streamId: "message"
-            type: FIELDS
-            args: ["key"]
-    -   name: "parser -> errors"
-        from: "parserBolt"
-        to: "errorIndexingBolt"
-        grouping:
-            streamId: "error"
-            type: SHUFFLE
-    -   name: "indexing -> errors"
-        from: "indexingBolt"
-        to: "errorIndexingBolt"
-        grouping:
-            streamId: "error"
-            type: SHUFFLE

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/yaf/test.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/yaf/test.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/yaf/test.yaml
new file mode 100644
index 0000000..021d3f8
--- /dev/null
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/yaf/test.yaml
@@ -0,0 +1,95 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: "yaf-test"
+config:
+    topology.workers: 1
+
+
+components:
+    -   id: "parser"
+        className: "org.apache.metron.parsing.parsers.GrokParser"
+        constructorArgs:
+            - "../Metron-MessageParsers/src/main/resources/patterns/yaf"
+            - "YAF_DELIMITED"
+        configMethods:
+            -   name: "withTimestampField"
+                args:
+                    - "start_time"
+            -   name: "withTimeFields"
+                args:
+                    - ["start_time", "end_time"]
+            -   name: "withDateFormat"
+                args:
+                    - "yyyy-MM-dd HH:mm:ss.S"
+    -   id: "writer"
+        className: "org.apache.metron.writer.KafkaWriter"
+        constructorArgs:
+            - "${kafka.broker}"
+    -   id: "zkHosts"
+        className: "storm.kafka.ZkHosts"
+        constructorArgs:
+            - "${kafka.zk}"
+    -   id: "kafkaConfig"
+        className: "storm.kafka.SpoutConfig"
+        constructorArgs:
+            # zookeeper hosts
+            - ref: "zkHosts"
+            # topic name
+            - "${spout.kafka.topic.yaf}"
+            # zk root
+            - ""
+            # id
+            - "${spout.kafka.topic.yaf}"
+        properties:
+            -   name: "ignoreZkOffsets"
+                value: true
+            -   name: "startOffsetTime"
+                value: -2
+            -   name: "socketTimeoutMs"
+                value: 1000000
+
+spouts:
+    -   id: "testingSpout"
+        className: "org.apache.metron.test.spouts.GenericInternalTestSpout"
+        parallelism: 1
+        configMethods:
+            -   name: "withFilename"
+                args:
+                    - "SampleInput/YafExampleOutput"
+            -   name: "withRepeating"
+                args:
+                    - false
+    -   id: "kafkaSpout"
+        className: "storm.kafka.KafkaSpout"
+        constructorArgs:
+            - ref: "kafkaConfig"
+
+bolts:
+    -   id: "parserBolt"
+        className: "org.apache.metron.bolt.ParserBolt"
+        constructorArgs:
+            - "${kafka.zk}"
+            - "yaf"
+            - ref: "parser"
+            - ref: "writer"
+
+streams:
+    -   name: "spout -> bolt"
+        from: "kafkaSpout"
+        to: "parserBolt"
+        grouping:
+            type: SHUFFLE

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/resources/SampleIndexed/YafIndexed
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/SampleIndexed/YafIndexed b/metron-streaming/Metron-Topologies/src/main/resources/SampleIndexed/YafIndexed
new file mode 100644
index 0000000..27b3589
--- /dev/null
+++ b/metron-streaming/Metron-Topologies/src/main/resources/SampleIndexed/YafIndexed
@@ -0,0 +1,10 @@
+{enrichments.geo.dip.longitude=test longitude, iflags=AS, enrichments.geo.dip.location_point=test longitude,test latitude, uflags=0, isn=22efa001, dip=10.0.2.15, dp=39468, threatintels.ip.dip=, enrichments.geo.sip.postalCode=test postalCode, duration=0.000, rpkt=0, enrichments.geo.dip.country=test country, original_string=2016-01-28 15:29:48.512|2016-01-28 15:29:48.512|   0.000|   0.000|  6|                          216.21.170.221|   80|                               10.0.2.15|39468|      AS|       0|       0|       0|22efa001|00000000|000|000|       1|      44|       0|       0|    0|idle, enrichments.geo.dip.locID=1, enrichments.geo.sip.city=test city, enrichments.geo.dip.latitude=test latitude, enrichments.geo.sip.country=test country, enrichments.geo.dip.city=test city, enrichments.geo.sip.dmaCode=test dmaCode, pkt=1, enrichments.geo.sip.location_point=test longitude,test latitude, ruflags=0, roct=0, sip=216.21.170.221, tag=0, enrichments.geo.dip.dmaCode=test dmaCode, rtag=0, sp
 =80, enrichments.geo.sip.longitude=test longitude, enrichments.geo.sip.latitude=test latitude, timestamp=1453994988512, app=0, threatintels.ip.sip=, oct=44, end_reason=idle, enrichments.geo.sip.locID=1, risn=0, enrichments.host.dip.known_info.type=printer, end_time=1453994988512, enrichments.host.dip.known_info.asset_value=important, enrichments.geo.dip.postalCode=test postalCode, source.type=yaf, enrichments.host.sip=, start_time=1453994988512, riflags=0, rtt=0.000, proto=6, enrichments.host.dip.known_info.local=YES}
+{enrichments.geo.dip.longitude=test longitude, iflags=A, enrichments.geo.dip.location_point=test longitude,test latitude, uflags=0, enrichments.host.sip.known_info.asset_value=important, isn=10000000, dip=10.0.2.3, enrichments.host.sip.known_info.local=YES, dp=53, enrichments.geo.sip.postalCode=test postalCode, duration=0.000, rpkt=0, enrichments.geo.dip.country=test country, original_string=2016-01-28 15:29:48.502|2016-01-28 15:29:48.502|   0.000|   0.000| 17|                               10.0.2.15|37299|                                10.0.2.3|   53|       A|       0|       0|       0|10000000|00000000|000|000|       1|      56|       0|       0|    0|idle, enrichments.geo.dip.locID=1, enrichments.geo.sip.city=test city, enrichments.host.sip.known_info.type=printer, enrichments.geo.dip.latitude=test latitude, enrichments.geo.sip.country=test country, enrichments.geo.dip.city=test city, enrichments.geo.sip.dmaCode=test dmaCode, pkt=1, enrichments.geo.sip.location_point=test longit
 ude,test latitude, ruflags=0, roct=0, sip=10.0.2.15, tag=0, enrichments.geo.dip.dmaCode=test dmaCode, rtag=0, sp=37299, enrichments.geo.sip.longitude=test longitude, enrichments.geo.sip.latitude=test latitude, timestamp=1453994988502, app=0, threatintels.ip.sip=, enrichments.host.dip=, oct=56, end_reason=idle, enrichments.geo.sip.locID=1, risn=0, end_time=1453994988502, enrichments.geo.dip.postalCode=test postalCode, source.type=yaf, start_time=1453994988502, riflags=0, rtt=0.000, threatintels.ip.dip.threat_source=ip_threat_intel, proto=17}
+{enrichments.geo.dip.longitude=test longitude, iflags=A, enrichments.geo.dip.location_point=test longitude,test latitude, uflags=0, isn=0, dip=10.0.2.15, dp=37299, threatintels.ip.dip=, enrichments.geo.sip.postalCode=test postalCode, duration=0.000, rpkt=0, enrichments.geo.dip.country=test country, original_string=2016-01-28 15:29:48.504|2016-01-28 15:29:48.504|   0.000|   0.000| 17|                                10.0.2.3|   53|                               10.0.2.15|37299|       A|       0|       0|       0|00000000|00000000|000|000|       1|     312|       0|       0|    0|idle, enrichments.geo.dip.locID=1, enrichments.geo.sip.city=test city, enrichments.geo.dip.latitude=test latitude, enrichments.geo.sip.country=test country, enrichments.geo.dip.city=test city, enrichments.geo.sip.dmaCode=test dmaCode, pkt=1, enrichments.geo.sip.location_point=test longitude,test latitude, ruflags=0, roct=0, sip=10.0.2.3, tag=0, enrichments.geo.dip.dmaCode=test dmaCode, rtag=0, sp=53, enrichmen
 ts.geo.sip.longitude=test longitude, enrichments.geo.sip.latitude=test latitude, timestamp=1453994988504, app=0, oct=312, end_reason=idle, enrichments.geo.sip.locID=1, risn=0, enrichments.host.dip.known_info.type=printer, end_time=1453994988504, enrichments.host.dip.known_info.asset_value=important, enrichments.geo.dip.postalCode=test postalCode, source.type=yaf, enrichments.host.sip=, start_time=1453994988504, threatintels.ip.sip.threat_source=ip_threat_intel, riflags=0, rtt=0.000, proto=17, enrichments.host.dip.known_info.local=YES}
+{enrichments.geo.dip.longitude=test longitude, iflags=A, enrichments.geo.dip.location_point=test longitude,test latitude, uflags=0, enrichments.host.sip.known_info.asset_value=important, isn=0, dip=10.0.2.3, enrichments.host.sip.known_info.local=YES, dp=53, enrichments.geo.sip.postalCode=test postalCode, duration=0.000, rpkt=0, enrichments.geo.dip.country=test country, original_string=2016-01-28 15:29:48.504|2016-01-28 15:29:48.504|   0.000|   0.000| 17|                               10.0.2.15|56303|                                10.0.2.3|   53|       A|       0|       0|       0|00000000|00000000|000|000|       1|      56|       0|       0|    0|idle, enrichments.geo.dip.locID=1, enrichments.geo.sip.city=test city, enrichments.host.sip.known_info.type=printer, enrichments.geo.dip.latitude=test latitude, enrichments.geo.sip.country=test country, enrichments.geo.dip.city=test city, enrichments.geo.sip.dmaCode=test dmaCode, pkt=1, enrichments.geo.sip.location_point=test longitude,tes
 t latitude, ruflags=0, roct=0, sip=10.0.2.15, tag=0, enrichments.geo.dip.dmaCode=test dmaCode, rtag=0, sp=56303, enrichments.geo.sip.longitude=test longitude, enrichments.geo.sip.latitude=test latitude, timestamp=1453994988504, app=0, threatintels.ip.sip=, enrichments.host.dip=, oct=56, end_reason=idle, enrichments.geo.sip.locID=1, risn=0, end_time=1453994988504, enrichments.geo.dip.postalCode=test postalCode, source.type=yaf, start_time=1453994988504, riflags=0, rtt=0.000, threatintels.ip.dip.threat_source=ip_threat_intel, proto=17}
+{enrichments.geo.dip.longitude=test longitude, iflags=A, enrichments.geo.dip.location_point=test longitude,test latitude, uflags=0, isn=0, dip=10.0.2.15, dp=56303, threatintels.ip.dip=, enrichments.geo.sip.postalCode=test postalCode, duration=0.000, rpkt=0, enrichments.geo.dip.country=test country, original_string=2016-01-28 15:29:48.506|2016-01-28 15:29:48.506|   0.000|   0.000| 17|                                10.0.2.3|   53|                               10.0.2.15|56303|       A|       0|       0|       0|00000000|00000000|000|000|       1|      84|       0|       0|    0|idle, enrichments.geo.dip.locID=1, enrichments.geo.sip.city=test city, enrichments.geo.dip.latitude=test latitude, enrichments.geo.sip.country=test country, enrichments.geo.dip.city=test city, enrichments.geo.sip.dmaCode=test dmaCode, pkt=1, enrichments.geo.sip.location_point=test longitude,test latitude, ruflags=0, roct=0, sip=10.0.2.3, tag=0, enrichments.geo.dip.dmaCode=test dmaCode, rtag=0, sp=53, enrichmen
 ts.geo.sip.longitude=test longitude, enrichments.geo.sip.latitude=test latitude, timestamp=1453994988506, app=0, oct=84, end_reason=idle, enrichments.geo.sip.locID=1, risn=0, enrichments.host.dip.known_info.type=printer, end_time=1453994988506, enrichments.host.dip.known_info.asset_value=important, enrichments.geo.dip.postalCode=test postalCode, source.type=yaf, enrichments.host.sip=, start_time=1453994988506, threatintels.ip.sip.threat_source=ip_threat_intel, riflags=0, rtt=0.000, proto=17, enrichments.host.dip.known_info.local=YES}
+{enrichments.geo.dip.longitude=test longitude, iflags=S, enrichments.geo.dip.location_point=test longitude,test latitude, uflags=0, enrichments.host.sip.known_info.asset_value=important, isn=58c52fca, dip=216.21.170.221, enrichments.host.sip.known_info.local=YES, dp=80, threatintels.ip.dip=, enrichments.geo.sip.postalCode=test postalCode, duration=0.000, rpkt=0, enrichments.geo.dip.country=test country, original_string=2016-01-28 15:29:48.508|2016-01-28 15:29:48.508|   0.000|   0.000|  6|                               10.0.2.15|39468|                          216.21.170.221|   80|       S|       0|       0|       0|58c52fca|00000000|000|000|       1|      60|       0|       0|    0|idle, enrichments.geo.dip.locID=1, enrichments.geo.sip.city=test city, enrichments.host.sip.known_info.type=printer, enrichments.geo.dip.latitude=test latitude, enrichments.geo.sip.country=test country, enrichments.geo.dip.city=test city, enrichments.geo.sip.dmaCode=test dmaCode, pkt=1, enrichments.geo.si
 p.location_point=test longitude,test latitude, ruflags=0, roct=0, sip=10.0.2.15, tag=0, enrichments.geo.dip.dmaCode=test dmaCode, rtag=0, sp=39468, enrichments.geo.sip.longitude=test longitude, enrichments.geo.sip.latitude=test latitude, timestamp=1453994988508, app=0, threatintels.ip.sip=, enrichments.host.dip=, oct=60, end_reason=idle, enrichments.geo.sip.locID=1, risn=0, end_time=1453994988508, enrichments.geo.dip.postalCode=test postalCode, source.type=yaf, start_time=1453994988508, riflags=0, rtt=0.000, proto=6}
+{enrichments.geo.dip.longitude=test longitude, iflags=A, enrichments.geo.dip.location_point=test longitude,test latitude, uflags=0, enrichments.host.sip.known_info.asset_value=important, isn=58c52fcb, dip=216.21.170.221, enrichments.host.sip.known_info.local=YES, dp=80, threatintels.ip.dip=, enrichments.geo.sip.postalCode=test postalCode, duration=0.000, rpkt=0, enrichments.geo.dip.country=test country, original_string=2016-01-28 15:29:48.512|2016-01-28 15:29:48.512|   0.000|   0.000|  6|                               10.0.2.15|39468|                          216.21.170.221|   80|       A|       0|       0|       0|58c52fcb|00000000|000|000|       1|      40|       0|       0|    0|idle , enrichments.geo.dip.locID=1, enrichments.geo.sip.city=test city, enrichments.host.sip.known_info.type=printer, enrichments.geo.dip.latitude=test latitude, enrichments.geo.sip.country=test country, enrichments.geo.dip.city=test city, enrichments.geo.sip.dmaCode=test dmaCode, pkt=1, enrichments.geo.s
 ip.location_point=test longitude,test latitude, ruflags=0, roct=0, sip=10.0.2.15, tag=0, enrichments.geo.dip.dmaCode=test dmaCode, rtag=0, sp=39468, enrichments.geo.sip.longitude=test longitude, enrichments.geo.sip.latitude=test latitude, timestamp=1453994988512, app=0, threatintels.ip.sip=, enrichments.host.dip=, oct=40, end_reason=idle , enrichments.geo.sip.locID=1, risn=0, end_time=1453994988512, enrichments.geo.dip.postalCode=test postalCode, source.type=yaf, start_time=1453994988512, riflags=0, rtt=0.000, proto=6}
+{enrichments.geo.dip.longitude=test longitude, iflags=AP, enrichments.geo.dip.location_point=test longitude,test latitude, uflags=0, enrichments.host.sip.known_info.asset_value=important, isn=58c52fcb, dip=216.21.170.221, enrichments.host.sip.known_info.local=YES, dp=80, threatintels.ip.dip=, enrichments.geo.sip.postalCode=test postalCode, duration=0.000, rpkt=0, enrichments.geo.dip.country=test country, original_string=2016-01-28 15:29:48.512|2016-01-28 15:29:48.512|   0.000|   0.000|  6|                               10.0.2.15|39468|                          216.21.170.221|   80|      AP|       0|       0|       0|58c52fcb|00000000|000|000|       1|     148|       0|       0|    0|idle , enrichments.geo.dip.locID=1, enrichments.geo.sip.city=test city, enrichments.host.sip.known_info.type=printer, enrichments.geo.dip.latitude=test latitude, enrichments.geo.sip.country=test country, enrichments.geo.dip.city=test city, enrichments.geo.sip.dmaCode=test dmaCode, pkt=1, enrichments.geo.
 sip.location_point=test longitude,test latitude, ruflags=0, roct=0, sip=10.0.2.15, tag=0, enrichments.geo.dip.dmaCode=test dmaCode, rtag=0, sp=39468, enrichments.geo.sip.longitude=test longitude, enrichments.geo.sip.latitude=test latitude, timestamp=1453994988512, app=0, threatintels.ip.sip=, enrichments.host.dip=, oct=148, end_reason=idle , enrichments.geo.sip.locID=1, risn=0, end_time=1453994988512, enrichments.geo.dip.postalCode=test postalCode, source.type=yaf, start_time=1453994988512, riflags=0, rtt=0.000, proto=6}
+{enrichments.geo.dip.longitude=test longitude, iflags=A, enrichments.geo.dip.location_point=test longitude,test latitude, uflags=0, isn=22efa002, dip=10.0.2.15, dp=39468, threatintels.ip.dip=, enrichments.geo.sip.postalCode=test postalCode, duration=0.000, rpkt=0, enrichments.geo.dip.country=test country, original_string=2016-01-28 15:29:48.512|2016-01-28 15:29:48.512|   0.000|   0.000|  6|                          216.21.170.221|   80|                               10.0.2.15|39468|       A|       0|       0|       0|22efa002|00000000|000|000|       1|      40|       0|       0|    0|idle , enrichments.geo.dip.locID=1, enrichments.geo.sip.city=test city, enrichments.geo.dip.latitude=test latitude, enrichments.geo.sip.country=test country, enrichments.geo.dip.city=test city, enrichments.geo.sip.dmaCode=test dmaCode, pkt=1, enrichments.geo.sip.location_point=test longitude,test latitude, ruflags=0, roct=0, sip=216.21.170.221, tag=0, enrichments.geo.dip.dmaCode=test dmaCode, rtag=0, sp
 =80, enrichments.geo.sip.longitude=test longitude, enrichments.geo.sip.latitude=test latitude, timestamp=1453994988512, app=0, threatintels.ip.sip=, oct=40, end_reason=idle , enrichments.geo.sip.locID=1, risn=0, enrichments.host.dip.known_info.type=printer, end_time=1453994988512, enrichments.host.dip.known_info.asset_value=important, enrichments.geo.dip.postalCode=test postalCode, source.type=yaf, enrichments.host.sip=, start_time=1453994988512, riflags=0, rtt=0.000, proto=6, enrichments.host.dip.known_info.local=YES}
+{enrichments.geo.dip.longitude=test longitude, iflags=AP, enrichments.geo.dip.location_point=test longitude,test latitude, uflags=0, isn=22efa002, dip=10.0.2.15, dp=39468, threatintels.ip.dip=, enrichments.geo.sip.postalCode=test postalCode, duration=0.000, rpkt=0, enrichments.geo.dip.country=test country, original_string=2016-01-28 15:29:48.562|2016-01-28 15:29:48.562|   0.000|   0.000|  6|                          216.21.170.221|   80|                               10.0.2.15|39468|      AP|       0|       0|       0|22efa002|00000000|000|000|       1|     604|       0|       0|    0|idle, enrichments.geo.dip.locID=1, enrichments.geo.sip.city=test city, enrichments.geo.dip.latitude=test latitude, enrichments.geo.sip.country=test country, enrichments.geo.dip.city=test city, enrichments.geo.sip.dmaCode=test dmaCode, pkt=1, enrichments.geo.sip.location_point=test longitude,test latitude, ruflags=0, roct=0, sip=216.21.170.221, tag=0, enrichments.geo.dip.dmaCode=test dmaCode, rtag=0, sp
 =80, enrichments.geo.sip.longitude=test longitude, enrichments.geo.sip.latitude=test latitude, timestamp=1453994988562, app=0, threatintels.ip.sip=, oct=604, end_reason=idle, enrichments.geo.sip.locID=1, risn=0, enrichments.host.dip.known_info.type=printer, end_time=1453994988562, enrichments.host.dip.known_info.asset_value=important, enrichments.geo.dip.postalCode=test postalCode, source.type=yaf, enrichments.host.sip=, start_time=1453994988562, riflags=0, rtt=0.000, proto=6, enrichments.host.dip.known_info.local=YES}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/resources/SampleInput/SnortOutput
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/SampleInput/SnortOutput b/metron-streaming/Metron-Topologies/src/main/resources/SampleInput/SnortOutput
index 2b9836e..0497b0f 100644
--- a/metron-streaming/Metron-Topologies/src/main/resources/SampleInput/SnortOutput
+++ b/metron-streaming/Metron-Topologies/src/main/resources/SampleInput/SnortOutput
@@ -1 +1,3 @@
-01/27-16:01:04.877970 ,129,12,1,"Consecutive TCP small segments exceeding threshold",TCP,10.0.2.2,56642,10.0.2.15,22,52:54:00:12:35:02,08:00:27:7F:93:2D,0x4E,***AP***,0x9AFF3D7,0xC8761D52,,0xFFFF,64,0,59677,64,65536,,,,
\ No newline at end of file
+01/27-16:01:04.877970 ,129,12,1,"Consecutive TCP small segments exceeding threshold",TCP,10.0.2.2,56642,10.0.2.15,22,52:54:00:12:35:02,08:00:27:7F:93:2D,0x4E,***AP***,0x9AFF3D7,0xC8761D52,,0xFFFF,64,0,59677,64,65536,,,,
+02/22-15:56:48.612494 ,129,12,1,"Consecutive TCP small segments exceeding threshold",TCP,96.44.142.5,80,10.0.2.15,50895,52:54:00:12:35:02,08:00:27:7F:93:2D,0x6E,***AP***,0xDB45F7A,0x7701DD5B,,0xFFFF,64,0,16785,96,98304,,,,
+02/22-15:56:48.616775 ,129,12,1,"Consecutive TCP small segments exceeding threshold",TCP,96.44.142.5,80,10.0.2.15,50895,52:54:00:12:35:02,08:00:27:7F:93:2D,0xA6,***AP***,0xDB508F2,0x7701DD5B,,0xFFFF,64,0,16824,152,155648,,,,
\ No newline at end of file


Mime
View raw message