metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From o...@apache.org
Subject [20/51] [abbrv] metron git commit: METRON-1750 Create Parser for Syslog RFC 5424 Messages (ottobackwards) closes apache/metron#1175
Date Wed, 24 Oct 2018 15:04:09 GMT
METRON-1750 Create Parser for Syslog RFC 5424 Messages (ottobackwards) closes apache/metron#1175


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/ff1f9cf5
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/ff1f9cf5
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/ff1f9cf5

Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: ff1f9cf52b31ae866eb3de1cf8993af97af1790f
Parents: c0fb262
Author: ottobackwards <ottobackwards@gmail.com>
Authored: Mon Oct 1 10:23:38 2018 -0400
Committer: otto <otto@apache.org>
Committed: Mon Oct 1 10:23:38 2018 -0400

----------------------------------------------------------------------
 dependencies_with_url.csv                       |   2 +-
 .../docker/rpm-docker/SPECS/metron.spec         |   3 +
 .../data/syslog5424/parsed/Syslog5424Parsed     |   3 +
 .../sample/data/syslog5424/raw/Syslog5424Output |   3 +
 metron-platform/metron-parsers/README.md        |   5 +-
 metron-platform/metron-parsers/pom.xml          |   5 +
 .../config/zookeeper/parsers/syslog5424.json    |   7 +
 .../metron/parsers/syslog/Syslog5424Parser.java | 102 +++++++++++++
 .../Syslog5424ParserIntegrationTest.java        |  37 +++++
 .../parsers/syslog/Syslog5424ParserTest.java    | 146 +++++++++++++++++++
 pom.xml                                         |   1 +
 11 files changed, 311 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/metron/blob/ff1f9cf5/dependencies_with_url.csv
----------------------------------------------------------------------
diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv
index ffd0fbc..53977f3 100644
--- a/dependencies_with_url.csv
+++ b/dependencies_with_url.csv
@@ -488,4 +488,4 @@ com.google.code.gson:gson:jar:2.8.2:compile,ASLv2,https://github.com/google/gson
   org.sonatype.sisu:sisu-inject-plexus:jar:2.2.2:compile
 com.zaxxer:HikariCP:jar:2.7.8:compile,ASLv2,https://github.com/brettwooldridge/HikariCP
 org.hibernate.validator:hibernate-validator:jar:6.0.9.Final:compile,ASLv2,https://github.com/hibernate/hibernate-validator
-
+com.github.palindromicity:simple-syslog-5424:jar:0.0.8:compile,ASLv2,https://github.com/palindromicity/simple-syslog-5424

http://git-wip-us.apache.org/repos/asf/metron/blob/ff1f9cf5/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
index 4bcef33..ed22a28 100644
--- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
+++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
@@ -158,6 +158,7 @@ This package installs the Metron Parser files
 %{metron_home}/config/zookeeper/parsers/jsonMapWrappedQuery.json
 %{metron_home}/config/zookeeper/parsers/snort.json
 %{metron_home}/config/zookeeper/parsers/squid.json
+%{metron_home}/config/zookeeper/parsers/syslog5424.json
 %{metron_home}/config/zookeeper/parsers/websphere.json
 %{metron_home}/config/zookeeper/parsers/yaf.json
 %{metron_home}/config/zookeeper/parsers/asa.json
@@ -590,6 +591,8 @@ chkconfig --del metron-alerts-ui
 %changelog
 * Thu Aug 30 2018 Apache Metron <dev@metron.apache.org> - 0.6.1
 - Update compiled css file name for Alerts UI
+* Fri Aug 24 2018 Apache Metron <dev@metron.apache.org> - 0.6.1
+- Add syslog5424 parser
 * Tue Aug 21 2018 Apache Metron <dev@metron.apache.org> - 0.6.1
 - Add Profiler for REPL
 * Tue Aug 14 2018 Apache Metron <dev@metron.apache.org> - 0.5.1

http://git-wip-us.apache.org/repos/asf/metron/blob/ff1f9cf5/metron-platform/metron-integration-test/src/main/sample/data/syslog5424/parsed/Syslog5424Parsed
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/sample/data/syslog5424/parsed/Syslog5424Parsed
b/metron-platform/metron-integration-test/src/main/sample/data/syslog5424/parsed/Syslog5424Parsed
new file mode 100644
index 0000000..e330204
--- /dev/null
+++ b/metron-platform/metron-integration-test/src/main/sample/data/syslog5424/parsed/Syslog5424Parsed
@@ -0,0 +1,3 @@
+{"syslog.header.appName":"d0602076-b14a-4c55-852a-981e7afeed38","syslog.header.version":"1","syslog.header.hostName":"loggregator","original_string":"<14>1
2014-06-20T09:14:07+00:00 loggregator d0602076-b14a-4c55-852a-981e7afeed38 DEA - - Removing
instance","syslog.header.facility":"1","syslog.header.msgId":"-","syslog.header.timestamp":"2014-06-20T09:14:07+00:00","syslog.message":"Removing
instance","syslog.header.pri":"14","syslog.header.procId":"DEA","syslog.header.severity":"6","timestamp":"2014-06-20T09:14:07+00:00","guid":"this-is-random-uuid-will-be-36-chars","source.type":"syslog5424"}
+{"syslog.structuredData.exampleSDID@32480.iut":"4","syslog.structuredData.exampleSDID@32480.eventSource":"Other
Application","syslog.header.hostName":"loggregator","syslog.header.facility":"1","syslog.structuredData.exampleSDID@32480.eventID":"2022","syslog.structuredData.exampleSDID@32473.eventSource":"Application","syslog.header.timestamp":"2014-06-20T09:14:08+00:00","syslog.message":"Removing
instance","syslog.header.pri":"14","syslog.header.procId":"DEA","syslog.header.severity":"6","syslog.header.appName":"d0602076-b14a-4c55-852a-981e7afeed40","syslog.header.version":"1","syslog.structuredData.exampleSDID@32473.iut":"3","original_string":"<14>1
2014-06-20T09:14:08+00:00 loggregator d0602076-b14a-4c55-852a-981e7afeed40 DEA MSG-02 [exampleSDID@32473
iut=\"3\" eventSource=\"Application\" eventID=\"1011\"] [exampleSDID@32480 iut=\"4\" eventSource=\"Other
Application\" eventID=\"2022\"] Removing instance","syslog.header.msgId":"MSG-02","syslog.structuredData.exampleSDID@32473.eventI
 D":"1011","timestamp":"2014-06-20T09:14:08+00:00","guid":"this-is-random-uuid-will-be-36-chars","source.type":"syslog5424"}
+{"syslog.structuredData.exampleSDID@32480.iut":"4","syslog.structuredData.exampleSDID@32480.eventSource":"Other
Application","syslog.structuredData.exampleSDID@32474.iut":"3","syslog.structuredData.exampleSDID@32474.eventID":"1011","syslog.header.hostName":"loggregator","syslog.header.facility":"1","syslog.structuredData.exampleSDID@32480.eventID":"2022","syslog.header.timestamp":"2014-06-20T09:14:09+00:00","syslog.message":"Removing
instance","syslog.header.pri":"14","syslog.header.procId":"DEA","syslog.structuredData.exampleSDID@32474.eventSource":"Application","syslog.header.severity":"6","syslog.header.appName":"d0602076-b14a-4c55-852a-981e7afeed42","syslog.header.version":"1","original_string":"<14>1
2014-06-20T09:14:09+00:00 loggregator d0602076-b14a-4c55-852a-981e7afeed42 DEA MSG-03 [exampleSDID@32474
iut=\"3\" eventSource=\"Application\" eventID=\"1011\"] [exampleSDID@32480 iut=\"4\" eventSource=\"Other
Application\" eventID=\"2022\"] Removing instance","syslog.header.msgId"
 :"MSG-03","timestamp":"2014-06-20T09:14:09+00:00","guid":"this-is-random-uuid-will-be-36-chars","source.type":"syslog5424"}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/metron/blob/ff1f9cf5/metron-platform/metron-integration-test/src/main/sample/data/syslog5424/raw/Syslog5424Output
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/sample/data/syslog5424/raw/Syslog5424Output
b/metron-platform/metron-integration-test/src/main/sample/data/syslog5424/raw/Syslog5424Output
new file mode 100644
index 0000000..869797a
--- /dev/null
+++ b/metron-platform/metron-integration-test/src/main/sample/data/syslog5424/raw/Syslog5424Output
@@ -0,0 +1,3 @@
+<14>1 2014-06-20T09:14:07+00:00 loggregator d0602076-b14a-4c55-852a-981e7afeed38 DEA
- - Removing instance
+<14>1 2014-06-20T09:14:08+00:00 loggregator d0602076-b14a-4c55-852a-981e7afeed40 DEA
MSG-02 [exampleSDID@32473 iut="3" eventSource="Application" eventID="1011"] [exampleSDID@32480
iut="4" eventSource="Other Application" eventID="2022"] Removing instance
+<14>1 2014-06-20T09:14:09+00:00 loggregator d0602076-b14a-4c55-852a-981e7afeed42 DEA
MSG-03 [exampleSDID@32474 iut="3" eventSource="Application" eventID="1011"] [exampleSDID@32480
iut="4" eventSource="Other Application" eventID="2022"] Removing instance
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/metron/blob/ff1f9cf5/metron-platform/metron-parsers/README.md
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/README.md b/metron-platform/metron-parsers/README.md
index fd6b470..381b0ee 100644
--- a/metron-platform/metron-parsers/README.md
+++ b/metron-platform/metron-parsers/README.md
@@ -24,7 +24,7 @@ Parsers are pluggable components which are used to transform raw data
 enrichment and indexing.  
 
 There are two general types types of parsers:
-*  A parser written in Java which conforms to the `MessageParser` interface.  This kind of
parser is optimized for speed and performance and is built for use with higher velocity topologies.
 These parsers are not easily modifiable and in order to make changes to them the entire topology
need to be recompiled.  
+* A parser written in Java which conforms to the `MessageParser` interface.  This kind of
parser is optimized for speed and performance and is built for use with higher velocity topologies.
 These parsers are not easily modifiable and in order to make changes to them the entire topology
need to be recompiled.  
 * A general purpose parser.  This type of parser is primarily designed for lower-velocity
topologies or for quickly standing up a parser for a new telemetry before a permanent Java
parser can be written for it.  As of the time of this writing, we have:
   * Grok parser: `org.apache.metron.parsers.GrokParser` with possible `parserConfig` entries
of 
     * `grokPath` : The path in HDFS (or in the Jar) to the grok statement
@@ -504,12 +504,13 @@ Parser adapters are loaded dynamically in each Metron topology.  They
 are defined in the Parser Config (defined above) JSON file in Zookeeper.
 
 ### Java Parser Adapters
-Java parser adapters are indended for higher-velocity topologies and are not easily changed
or extended.  As the adoption of Metron continues we plan on extending our library of Java
adapters to process more log formats.  As of this moment the Java adapters included with Metron
are:
+Java parser adapters are intended for higher-velocity topologies and are not easily changed
or extended.  As the adoption of Metron continues we plan on extending our library of Java
adapters to process more log formats.  As of this moment the Java adapters included with Metron
are:
 
 * org.apache.metron.parsers.ise.BasicIseParser : Parse ISE messages
 * org.apache.metron.parsers.bro.BasicBroParser : Parse Bro messages
 * org.apache.metron.parsers.sourcefire.BasicSourcefireParser : Parse Sourcefire messages
 * org.apache.metron.parsers.lancope.BasicLancopeParser : Parse Lancope messages
+* org.apache.metron.parsers.syslog.Syslog5424Parser : Parse Syslog RFC 5424 messages
 
 ### Grok Parser Adapters
 Grok parser adapters are designed primarly for someone who is not a Java coder for quickly
standing up a parser adapter for lower velocity topologies.  Grok relies on Regex for message
parsing, which is much slower than purpose-built Java parsers, but is more extensible.  Grok
parsers are defined via a config file and the topplogy does not need to be recombiled in order
to make changes to them.  An example of a Grok perser is:

http://git-wip-us.apache.org/repos/asf/metron/blob/ff1f9cf5/metron-platform/metron-parsers/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/pom.xml b/metron-platform/metron-parsers/pom.xml
index e586492..5077791 100644
--- a/metron-platform/metron-parsers/pom.xml
+++ b/metron-platform/metron-parsers/pom.xml
@@ -261,6 +261,11 @@
             <artifactId>json-path</artifactId>
             <version>2.3.0</version>
         </dependency>
+        <dependency>
+            <groupId>com.github.palindromicity</groupId>
+            <artifactId>simple-syslog-5424</artifactId>
+            <version>${global_simple_syslog_version}</version>
+        </dependency>
     </dependencies>
     <build>
         <plugins>

http://git-wip-us.apache.org/repos/asf/metron/blob/ff1f9cf5/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/syslog5424.json
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/syslog5424.json
b/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/syslog5424.json
new file mode 100644
index 0000000..5f62692
--- /dev/null
+++ b/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/syslog5424.json
@@ -0,0 +1,7 @@
+{
+  "parserClassName":"org.apache.metron.parsers.syslog.Syslog5424Parser",
+  "sensorTopic":"syslog5424",
+  "parserConfig": {
+    "nilPolicy": "DASH"
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/metron/blob/ff1f9cf5/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/syslog/Syslog5424Parser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/syslog/Syslog5424Parser.java
b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/syslog/Syslog5424Parser.java
new file mode 100644
index 0000000..e3ad941
--- /dev/null
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/syslog/Syslog5424Parser.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.parsers.syslog;
+
+import com.github.palindromicity.syslog.AllowableDeviations;
+import com.github.palindromicity.syslog.NilPolicy;
+import com.github.palindromicity.syslog.SyslogParser;
+import com.github.palindromicity.syslog.SyslogParserBuilder;
+import com.github.palindromicity.syslog.dsl.SyslogFieldKeys;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.metron.parsers.BasicParser;
+import org.json.simple.JSONObject;
+
+import java.io.BufferedReader;
+import java.io.Reader;
+import java.io.StringReader;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+
+
+/**
+ * Parser for well structured RFC 5424 messages.
+ */
+public class Syslog5424Parser extends BasicParser {
+  public static final String NIL_POLICY_CONFIG = "nilPolicy";
+  private transient SyslogParser syslogParser;
+
+  @Override
+  public void configure(Map<String, Object> config) {
+    // Default to OMIT policy for nil fields
+    // this means they will not be in the returned field set
+    String nilPolicyStr = (String) config.getOrDefault(NIL_POLICY_CONFIG, NilPolicy.OMIT.name());
+    NilPolicy nilPolicy = NilPolicy.valueOf(nilPolicyStr);
+    syslogParser = new SyslogParserBuilder()
+            .withNilPolicy(nilPolicy)
+            .withDeviations(EnumSet.of(AllowableDeviations.PRIORITY,AllowableDeviations.VERSION))
+            .build();
+  }
+
+  @Override
+  public void init() {
+  }
+
+  @Override
+  @SuppressWarnings("unchecked")
+  public List<JSONObject> parse(byte[] rawMessage) {
+    try {
+      if (rawMessage == null || rawMessage.length == 0) {
+        return null;
+      }
+
+      String originalString = new String(rawMessage);
+      List<JSONObject> returnList = new ArrayList<>();
+      try (Reader reader = new BufferedReader(new StringReader(originalString))) {
+        syslogParser.parseLines(reader, (m) -> {
+          JSONObject jsonObject = new JSONObject(m);
+          // be sure to put in the original string, and the timestamp.
+          // we wil just copy over the timestamp from the syslog
+          jsonObject.put("original_string", originalString);
+          setTimestamp(jsonObject);
+          returnList.add(jsonObject);
+        });
+
+        return returnList;
+      }
+    } catch (Exception e) {
+      String message = "Unable to parse " + new String(rawMessage) + ": " + e.getMessage();
+      LOG.error(message, e);
+      throw new IllegalStateException(message, e);
+    }
+  }
+
+  @SuppressWarnings("unchecked")
+  private void setTimestamp(JSONObject message) {
+    String timeStampString = (String) message.get(SyslogFieldKeys.HEADER_TIMESTAMP.getField());
+    if (!StringUtils.isBlank(timeStampString) && !timeStampString.equals("-")) {
+      message.put("timestamp", timeStampString);
+    } else {
+      message.put("timestamp", LocalDateTime.now().format(DateTimeFormatter.ISO_DATE_TIME));
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/ff1f9cf5/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/Syslog5424ParserIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/Syslog5424ParserIntegrationTest.java
b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/Syslog5424ParserIntegrationTest.java
new file mode 100644
index 0000000..c9c47ce
--- /dev/null
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/Syslog5424ParserIntegrationTest.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.parsers.integration;
+
+import org.apache.metron.parsers.integration.validation.SampleDataValidation;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class Syslog5424ParserIntegrationTest extends ParserIntegrationTest {
+  @Override
+  String getSensorType() {
+    return "syslog5424";
+  }
+
+  @Override
+  List<ParserValidation> getValidations() {
+    return new ArrayList<ParserValidation>() {{
+      add(new SampleDataValidation());
+    }};
+  }
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/ff1f9cf5/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/syslog/Syslog5424ParserTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/syslog/Syslog5424ParserTest.java
b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/syslog/Syslog5424ParserTest.java
new file mode 100644
index 0000000..0ef26ff
--- /dev/null
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/syslog/Syslog5424ParserTest.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.parsers.syslog;
+
+import com.github.palindromicity.syslog.NilPolicy;
+import com.github.palindromicity.syslog.dsl.SyslogFieldKeys;
+import org.json.simple.JSONObject;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.time.format.DateTimeFormatter;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Consumer;
+
+public class Syslog5424ParserTest {
+  private static final String SYSLOG_LINE_ALL = "<14>1 2014-06-20T09:14:07+00:00 loggregator"
+          + " d0602076-b14a-4c55-852a-981e7afeed38 DEA MSG-01"
+          + " [exampleSDID@32473 iut=\"3\" eventSource=\"Application\" eventID=\"1011\"]"
+          + " [exampleSDID@32480 iut=\"4\" eventSource=\"Other Application\" eventID=\"2022\"]
Removing instance";
+
+  private static final String SYSLOG_LINE_MISSING = "<14>1 2014-06-20T09:14:07+00:00
loggregator"
+          + " d0602076-b14a-4c55-852a-981e7afeed38 DEA -"
+          + " [exampleSDID@32473 iut=\"3\" eventSource=\"Application\" eventID=\"1011\"]"
+          + " [exampleSDID@32480 iut=\"4\" eventSource=\"Other Application\" eventID=\"2022\"]
Removing instance";
+
+  private static final String SYSLOG_LINE_MISSING_DATE = "<14>1 - loggregator"
+          + " d0602076-b14a-4c55-852a-981e7afeed38 DEA -"
+          + " [exampleSDID@32473 iut=\"3\" eventSource=\"Application\" eventID=\"1011\"]"
+          + " [exampleSDID@32480 iut=\"4\" eventSource=\"Other Application\" eventID=\"2022\"]
Removing instance";
+
+  private static final String expectedVersion = "1";
+  private static final String expectedMessage = "Removing instance";
+  private static final String expectedAppName = "d0602076-b14a-4c55-852a-981e7afeed38";
+  private static final String expectedHostName = "loggregator";
+  private static final String expectedPri = "14";
+  private static final String expectedFacility = "1";
+  private static final String expectedSeverity = "6";
+  private static final String expectedProcId = "DEA";
+  private static final String expectedTimestamp = "2014-06-20T09:14:07+00:00";
+  private static final String expectedMessageId = "MSG-01";
+
+  private static final String expectedIUT1 = "3";
+  private static final String expectedIUT2 = "4";
+  private static final String expectedEventSource1 = "Application";
+  private static final String expectedEventSource2 = "Other Application";
+  private static final String expectedEventID1 = "1011";
+  private static final String expectedEventID2 = "2022";
+
+  @Test
+  public void testHappyPath() {
+    test(null, SYSLOG_LINE_ALL, (message) -> Assert.assertEquals(expectedMessageId, message.get(SyslogFieldKeys.HEADER_MSGID.getField())));
+  }
+
+  @Test
+  public void testOmit() {
+    test(NilPolicy.OMIT, SYSLOG_LINE_MISSING, (message) -> Assert.assertFalse(message.containsKey(SyslogFieldKeys.HEADER_MSGID)));
+  }
+
+  @Test
+  public void testDash() {
+    test(NilPolicy.DASH, SYSLOG_LINE_MISSING, (message) -> Assert.assertEquals("-", message.get(SyslogFieldKeys.HEADER_MSGID.getField())));
+  }
+
+  @Test()
+  public void testNull() {
+    test(NilPolicy.NULL, SYSLOG_LINE_MISSING, (message) -> {
+      Assert.assertTrue(message.containsKey(SyslogFieldKeys.HEADER_MSGID.getField()));
+      Assert.assertNull(message.get(SyslogFieldKeys.HEADER_MSGID.getField()));
+    });
+  }
+
+  @Test(expected = IllegalStateException.class)
+  public void testNotValid() {
+    test(null, "not valid", (message) -> Assert.assertTrue(false));
+  }
+
+  public void test(NilPolicy nilPolicy, String line, Consumer<JSONObject> msgIdChecker)
{
+    Syslog5424Parser parser = new Syslog5424Parser();
+    Map<String, Object> config = new HashMap<>();
+    if (nilPolicy != null) {
+      config.put(Syslog5424Parser.NIL_POLICY_CONFIG, nilPolicy.name());
+    }
+    parser.configure(config);
+
+    List<JSONObject> output = parser.parse(line.getBytes());
+  }
+
+  @Test
+  public void testReadMultiLine() throws Exception {
+    Syslog5424Parser parser = new Syslog5424Parser();
+    Map<String, Object> config = new HashMap<>();
+    config.put(Syslog5424Parser.NIL_POLICY_CONFIG, NilPolicy.DASH.name());
+    parser.configure(config);
+    StringBuilder builder = new StringBuilder();
+    builder
+            .append(SYSLOG_LINE_ALL)
+            .append("\n")
+            .append(SYSLOG_LINE_MISSING)
+            .append("\n")
+            .append(SYSLOG_LINE_ALL);
+    List<JSONObject> output = parser.parse(builder.toString().getBytes());
+    Assert.assertEquals(3,output.size());
+  }
+
+  @Test
+  public void testMissingTimestamp() {
+    Syslog5424Parser parser = new Syslog5424Parser();
+    Map<String, Object> config = new HashMap<>();
+    config.put(Syslog5424Parser.NIL_POLICY_CONFIG, NilPolicy.DASH.name());
+    parser.configure(config);
+    List<JSONObject> output = parser.parse(SYSLOG_LINE_MISSING_DATE.getBytes());
+    String timeStampString = output.get(0).get("timestamp").toString();
+    DateTimeFormatter.ISO_DATE_TIME.parse(timeStampString);
+    config.clear();
+    config.put(Syslog5424Parser.NIL_POLICY_CONFIG, NilPolicy.NULL.name());
+    parser.configure(config);
+    output = parser.parse(SYSLOG_LINE_MISSING_DATE.getBytes());
+    timeStampString = output.get(0).get("timestamp").toString();
+    DateTimeFormatter.ISO_DATE_TIME.parse(timeStampString);
+
+    config.clear();
+    config.put(Syslog5424Parser.NIL_POLICY_CONFIG, NilPolicy.OMIT.name());
+    parser.configure(config);
+    output = parser.parse(SYSLOG_LINE_MISSING_DATE.getBytes());
+    timeStampString = output.get(0).get("timestamp").toString();
+    DateTimeFormatter.ISO_DATE_TIME.parse(timeStampString);
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/metron/blob/ff1f9cf5/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 1c1fa3d..9bde04e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -137,6 +137,7 @@
         <global_reflections_version>0.9.10</global_reflections_version>
         <global_checkstyle_version>8.0</global_checkstyle_version>
         <global_log4j_core_version>2.1</global_log4j_core_version>
+        <global_simple_syslog_version>0.0.8</global_simple_syslog_version>
         <global_spark_version>2.3.1</global_spark_version>
     </properties>
 


Mime
View raw message