metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ceste...@apache.org
Subject [29/30] incubator-metron git commit: METRON-21 Extend the ansible scripts to support vagrant
Date Fri, 29 Jan 2016 20:04:42 GMT
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/54c8de38/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/yaf/local.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/yaf/local.yaml
b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/yaf/local.yaml
new file mode 100644
index 0000000..e2ad01b
--- /dev/null
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/yaf/local.yaml
@@ -0,0 +1,185 @@
+name: "yaf-local"
+config:
+    topology.workers: 1
+
+components:
+    -   id: "yafParser"
+        className: "org.apache.metron.parsing.parsers.BasicYafParser"
+    -   id: "genericMessageFilter"
+        className: "org.apache.metron.filters.GenericMessageFilter"
+    -   id: "indexAdapter"
+        className: "org.apache.metron.indexing.adapters.ESTimedRotatingAdapter"
+    -   id: "metricConfig"
+        className: "org.apache.commons.configuration.BaseConfiguration"
+        configMethods:
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.graphite"
+                    - "${org.apache.metron.metrics.reporter.graphite}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.console"
+                    - "${org.apache.metron.metrics.reporter.console}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.jmx"
+                    - "${org.apache.metron.metrics.reporter.jmx}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.graphite.address"
+                    - "${org.apache.metron.metrics.graphite.address}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.graphite.port"
+                    - "${org.apache.metron.metrics.graphite.port}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.acks"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.emits"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.fails"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.fails}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.acks"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.emits"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.fails"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.fails}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.acks"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.emits"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.fails"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.fails}"
+
+spouts:
+    -   id: "testingSpout"
+        className: "org.apache.metron.test.spouts.GenericInternalTestSpout"
+        parallelism: 1
+        configMethods:
+            -   name: "withFilename"
+                args:
+                    - "SampleInput/YafExampleOutput"
+            -   name: "withRepeating"
+                args:
+                    - true
+
+bolts:
+    -   id: "parserBolt"
+        className: "org.apache.metron.parsing.TelemetryParserBolt"
+        configMethods:
+            -   name: "withMessageParser"
+                args:
+                    - ref: "yafParser"
+            -   name: "withOutputFieldName"
+                args:
+                    - "yaf"
+            -   name: "withMessageFilter"
+                args:
+                    - ref: "genericMessageFilter"
+            -   name: "withMetricConfig"
+                args:
+                    - ref: "metricConfig"
+    -   id: "indexingBolt"
+        className: "org.apache.metron.indexing.TelemetryIndexingBolt"
+        configMethods:
+            -   name: "withIndexIP"
+                args:
+                    - "${es.ip}"
+            -   name: "withIndexPort"
+                args:
+                    - ${es.port}
+            -   name: "withClusterName"
+                args:
+                    - "${es.clustername}"
+            -   name: "withIndexName"
+                args:
+                    - "yaf_index"
+            -   name: "withIndexTimestamp"
+                args:
+                    - "yyyy.MM.dd.hh"
+            -   name: "withDocumentName"
+                args:
+                    - "yaf_doc"
+            -   name: "withBulk"
+                args:
+                    - 1
+            -   name: "withIndexAdapter"
+                args:
+                    - ref: "indexAdapter"
+            -   name: "withMetricConfiguration"
+                args:
+                    - ref: "metricConfig"
+    -   id: "errorIndexingBolt"
+        className: "org.apache.metron.indexing.TelemetryIndexingBolt"
+        configMethods:
+            -   name: "withIndexIP"
+                args:
+                    - "${es.ip}"
+            -   name: "withIndexPort"
+                args:
+                    - ${es.port}
+            -   name: "withClusterName"
+                args:
+                    - "${es.clustername}"
+            -   name: "withIndexName"
+                args:
+                    - "error"
+            -   name: "withIndexTimestamp"
+                args:
+                    - "yyyy.MM"
+            -   name: "withDocumentName"
+                args:
+                    - "yaf_error"
+            -   name: "withBulk"
+                args:
+                    - 1
+            -   name: "withIndexAdapter"
+                args:
+                    - ref: "indexAdapter"
+            -   name: "withMetricConfiguration"
+                args:
+                    - ref: "metricConfig"
+
+streams:
+    -   name: "spout -> parser"
+        from: "testingSpout"
+        to: "parserBolt"
+        grouping:
+            type: SHUFFLE
+    -   name: "parser -> indexing"
+        from: "parserBolt"
+        to: "indexingBolt"
+        grouping:
+            streamId: "message"
+            type: FIELDS
+            args: ["key"]
+    -   name: "parser -> errors"
+        from: "parserBolt"
+        to: "errorIndexingBolt"
+        grouping:
+            streamId: "error"
+            type: SHUFFLE
+    -   name: "indexing -> errors"
+        from: "indexingBolt"
+        to: "errorIndexingBolt"
+        grouping:
+            streamId: "error"
+            type: SHUFFLE

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/54c8de38/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/yaf/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/yaf/remote.yaml
b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/yaf/remote.yaml
new file mode 100644
index 0000000..d4c1dd2
--- /dev/null
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/yaf/remote.yaml
@@ -0,0 +1,199 @@
+name: "yaf"
+config:
+    topology.workers: 1
+
+components:
+    -   id: "yafParser"
+        className: "org.apache.metron.parsing.parsers.BasicYafParser"
+    -   id: "genericMessageFilter"
+        className: "org.apache.metron.filters.GenericMessageFilter"
+    -   id: "indexAdapter"
+        className: "org.apache.metron.indexing.adapters.ESTimedRotatingAdapter"
+    -   id: "metricConfig"
+        className: "org.apache.commons.configuration.BaseConfiguration"
+        configMethods:
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.graphite"
+                    - "${org.apache.metron.metrics.reporter.graphite}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.console"
+                    - "${org.apache.metron.metrics.reporter.console}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.jmx"
+                    - "${org.apache.metron.metrics.reporter.jmx}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.graphite.address"
+                    - "${org.apache.metron.metrics.graphite.address}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.graphite.port"
+                    - "${org.apache.metron.metrics.graphite.port}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.acks"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.emits"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.fails"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.fails}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.acks"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.emits"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.fails"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.fails}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.acks"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.emits"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.fails"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.fails}"
+    -   id: "zkHosts"
+        className: "storm.kafka.ZkHosts"
+        constructorArgs:
+            - "${kafka.zk}"
+    -   id: "kafkaConfig"
+        className: "storm.kafka.SpoutConfig"
+        constructorArgs:
+            # zookeeper hosts
+            - ref: "zkHosts"
+            # topic name
+            - "${spout.kafka.topic.yaf}"
+            # zk root
+            - ""
+            # id
+            - "${spout.kafka.topic.yaf}"
+        properties:
+            -   name: "forceFromStart"
+                value: true
+            -   name: "startOffsetTime"
+                value: -1
+
+spouts:
+    -   id: "kafkaSpout"
+        className: "storm.kafka.KafkaSpout"
+        constructorArgs:
+            - ref: "kafkaConfig"
+
+bolts:
+    -   id: "parserBolt"
+        className: "org.apache.metron.parsing.TelemetryParserBolt"
+        configMethods:
+            -   name: "withMessageParser"
+                args:
+                    - ref: "yafParser"
+            -   name: "withOutputFieldName"
+                args:
+                    - "yaf"
+            -   name: "withMessageFilter"
+                args:
+                    - ref: "genericMessageFilter"
+            -   name: "withMetricConfig"
+                args:
+                    - ref: "metricConfig"
+    -   id: "indexingBolt"
+        className: "org.apache.metron.indexing.TelemetryIndexingBolt"
+        configMethods:
+            -   name: "withIndexIP"
+                args:
+                    - "${es.ip}"
+            -   name: "withIndexPort"
+                args:
+                    - ${es.port}
+            -   name: "withClusterName"
+                args:
+                    - "${es.clustername}"
+            -   name: "withIndexName"
+                args:
+                    - "yaf_index"
+            -   name: "withIndexTimestamp"
+                args:
+                    - "yyyy.MM.dd.hh"
+            -   name: "withDocumentName"
+                args:
+                    - "yaf_doc"
+            -   name: "withBulk"
+                args:
+                    - 1
+            -   name: "withIndexAdapter"
+                args:
+                    - ref: "indexAdapter"
+            -   name: "withMetricConfiguration"
+                args:
+                    - ref: "metricConfig"
+    -   id: "errorIndexingBolt"
+        className: "org.apache.metron.indexing.TelemetryIndexingBolt"
+        configMethods:
+            -   name: "withIndexIP"
+                args:
+                    - "${es.ip}"
+            -   name: "withIndexPort"
+                args:
+                    - ${es.port}
+            -   name: "withClusterName"
+                args:
+                    - "${es.clustername}"
+            -   name: "withIndexName"
+                args:
+                    - "error"
+            -   name: "withIndexTimestamp"
+                args:
+                    - "yyyy.MM"
+            -   name: "withDocumentName"
+                args:
+                    - "yaf_error"
+            -   name: "withBulk"
+                args:
+                    - 1
+            -   name: "withIndexAdapter"
+                args:
+                    - ref: "indexAdapter"
+            -   name: "withMetricConfiguration"
+                args:
+                    - ref: "metricConfig"
+
+streams:
+    -   name: "spout -> parser"
+        from: "kafkaSpout"
+        to: "parserBolt"
+        grouping:
+            type: SHUFFLE
+    -   name: "parser -> indexing"
+        from: "parserBolt"
+        to: "indexingBolt"
+        grouping:
+            streamId: "message"
+            type: FIELDS
+            args: ["key"]
+    -   name: "parser -> errors"
+        from: "parserBolt"
+        to: "errorIndexingBolt"
+        grouping:
+            streamId: "error"
+            type: SHUFFLE
+    -   name: "indexing -> errors"
+        from: "indexingBolt"
+        to: "errorIndexingBolt"
+        grouping:
+            streamId: "error"
+            type: SHUFFLE

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/54c8de38/metron-streaming/Metron-Topologies/src/main/resources/SampleInput/SnortOutput
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/SampleInput/SnortOutput
b/metron-streaming/Metron-Topologies/src/main/resources/SampleInput/SnortOutput
new file mode 100644
index 0000000..2b9836e
--- /dev/null
+++ b/metron-streaming/Metron-Topologies/src/main/resources/SampleInput/SnortOutput
@@ -0,0 +1 @@
+01/27-16:01:04.877970 ,129,12,1,"Consecutive TCP small segments exceeding threshold",TCP,10.0.2.2,56642,10.0.2.15,22,52:54:00:12:35:02,08:00:27:7F:93:2D,0x4E,***AP***,0x9AFF3D7,0xC8761D52,,0xFFFF,64,0,59677,64,65536,,,,
\ No newline at end of file


Mime
View raw message