metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From l...@apache.org
Subject incubator-metron git commit: METRON-242 remove Squid pattern closes apache/incubator-metron#202
Date Mon, 01 Aug 2016 14:56:33 GMT
Repository: incubator-metron
Updated Branches:
  refs/heads/master 9c70a44c7 -> bbd6474e2


METRON-242 remove Squid pattern closes apache/incubator-metron#202


Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/bbd6474e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/bbd6474e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/bbd6474e

Branch: refs/heads/master
Commit: bbd6474e2e48a39ac1471bbcd5f12e461d7ec9d8
Parents: 9c70a44
Author: dlyle65535 <dlyle65535@gmail.com>
Authored: Mon Aug 1 10:55:43 2016 -0400
Committer: David Lyle <dlyle65535@gmail.com>
Committed: Mon Aug 1 10:55:43 2016 -0400

----------------------------------------------------------------------
 .../src/main/resources/patterns/squid           |  3 +-
 .../apache/metron/parsers/GrokParserTest.java   | 36 ++++++++--------
 .../metron/parsers/SampleGrokParserTest.java    | 17 +++++---
 .../apache/metron/parsers/SquidParserTest.java  | 43 +++++++++++++++-----
 .../apache/metron/parsers/YafParserTest.java    | 30 ++++++++------
 5 files changed, 83 insertions(+), 46 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/bbd6474e/metron-platform/metron-parsers/src/main/resources/patterns/squid
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/resources/patterns/squid b/metron-platform/metron-parsers/src/main/resources/patterns/squid
index d7312cc..bf6c5b7 100644
--- a/metron-platform/metron-parsers/src/main/resources/patterns/squid
+++ b/metron-platform/metron-parsers/src/main/resources/patterns/squid
@@ -1 +1,2 @@
-SQUID_DELIMITED %{NUMBER:timestamp} %{SPACE:UNWANTED}  %{INT:elapsed} %{IPV4:ip_src_addr}
%{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url} - %{WORD:UNWANTED}\/%{IPV4:ip_dst_addr}
%{WORD:UNWANTED}\/%{WORD:UNWANTED}
\ No newline at end of file
+SQUID_DELIMITED %{NUMBER:timestamp}[^0-9]*%{INT:elapsed} %{IP:ip_src_addr} %{WORD:action}/%{NUMBER:code}
%{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url}[^0-9]*(%{IP:ip_dst_addr})?
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/bbd6474e/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/GrokParserTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/GrokParserTest.java
b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/GrokParserTest.java
index b1deb0e..9590d34 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/GrokParserTest.java
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/GrokParserTest.java
@@ -23,37 +23,42 @@ import junit.framework.Assert;
 import org.json.simple.JSONObject;
 import org.json.simple.parser.JSONParser;
 import org.json.simple.parser.ParseException;
-import org.junit.Before;
 import org.junit.Test;
 
 import java.io.IOException;
-import java.util.*;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
 public abstract class GrokParserTest {
 
-  private JSONObject expectedParsed;
-
-  @Before
-  public void parseJSON() throws ParseException {
-    JSONParser jsonParser = new JSONParser();
-    expectedParsed = (JSONObject) jsonParser.parse(getExpectedParsedString());
-  }
-
   @Test
   public void test() throws IOException, ParseException {
+
     Map<String, Object> parserConfig = new HashMap<>();
     parserConfig.put("grokPath", getGrokPath());
     parserConfig.put("patternLabel", getGrokPatternLabel());
     parserConfig.put("timestampField", getTimestampField());
     parserConfig.put("dateFormat", getDateFormat());
     parserConfig.put("timeFields", getTimeFields());
+
     GrokParser grokParser = new GrokParser();
     grokParser.configure(parserConfig);
     grokParser.init();
-    byte[] rawMessage = getRawMessage().getBytes();
-    List<JSONObject> parsedList = grokParser.parse(rawMessage);
-    Assert.assertEquals(1, parsedList.size());
-    compare(expectedParsed, parsedList.get(0));
+
+    JSONParser jsonParser = new JSONParser();
+    Map<String,String> testData = getTestData();
+    for( Map.Entry<String,String> e : testData.entrySet() ){
+
+      JSONObject expected = (JSONObject) jsonParser.parse(e.getValue());
+      byte[] rawMessage = e.getKey().getBytes();
+
+      List<JSONObject> parsedList = grokParser.parse(rawMessage);
+      Assert.assertEquals(1, parsedList.size());
+      compare(expected, parsedList.get(0));
+    }
+
   }
 
   public boolean compare(JSONObject expected, JSONObject actual) {
@@ -81,8 +86,7 @@ public abstract class GrokParserTest {
     return true;
   }
 
-  public abstract String getRawMessage();
-  public abstract String getExpectedParsedString();
+  public abstract Map getTestData();
   public abstract String getGrokPath();
   public abstract String getGrokPatternLabel();
   public abstract List<String> getTimeFields();

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/bbd6474e/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SampleGrokParserTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SampleGrokParserTest.java
b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SampleGrokParserTest.java
index fe11c3f..89eb30c 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SampleGrokParserTest.java
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SampleGrokParserTest.java
@@ -20,7 +20,9 @@ package org.apache.metron.parsers;
 import org.adrianwalker.multilinestring.Multiline;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 public class SampleGrokParserTest extends GrokParserTest {
 
@@ -53,14 +55,17 @@ public class SampleGrokParserTest extends GrokParserTest {
    * }
    */
   @Multiline
-  public String expectedParsedString;
+  public String result;
 
-  public String getExpectedParsedString() {
-    return expectedParsedString;
-  }
 
-  public String getRawMessage() {
-    return "1453994987000|2016-01-28 15:29:48|   0.000|   0.000|  6|                    
     216.21.170.221|   80|                               10.0.2.15|39468|      AS|       0|
      0|       0|22efa001|00000000|000|000|       1|      44|       0|       0|    0|idle";
+  @Override
+  public Map getTestData() {
+
+    Map testData = new HashMap<String,String>();
+    String input = "1453994987000|2016-01-28 15:29:48|   0.000|   0.000|  6|            
             216.21.170.221|   80|                               10.0.2.15|39468|      AS|
      0|       0|       0|22efa001|00000000|000|000|       1|      44|       0|       0| 
  0|idle";
+    testData.put(input,result);
+    return testData;
+
   }
 
   public String getGrokPath() {

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/bbd6474e/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SquidParserTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SquidParserTest.java
b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SquidParserTest.java
index 7f0abaf..93c8276 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SquidParserTest.java
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SquidParserTest.java
@@ -20,37 +20,60 @@ package org.apache.metron.parsers;
 import org.adrianwalker.multilinestring.Multiline;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 public class SquidParserTest extends GrokParserTest {
 
-  @Override
-  public String getRawMessage() {
-    return "1461576382.642    161 127.0.0.1 TCP_MISS/200 103701 GET http://www.cnn.com/ -
DIRECT/199.27.79.73 text/html";
-  }
-
   /**
    * {
    *   "elapsed":161,
    *   "code":200,
+   *   "ip_dst_addr":"199.27.79.73",
    *   "original_string":"1461576382.642    161 127.0.0.1 TCP_MISS/200 103701 GET http://www.cnn.com/
- DIRECT/199.27.79.73 text/html",
    *   "method":"GET",
    *   "bytes":103701,
    *   "action":"TCP_MISS",
-   *   "ip_src_addr":"127.0.0.1",
-   *   "ip_dst_addr":"199.27.79.73",
    *   "url":"http://www.cnn.com/",
+   *   "ip_src_addr":"127.0.0.1",
    *   "timestamp":1461576382642
    * }
    */
   @Multiline
-  public String expectedParsedString;
+  private String result1;
+
+  /**
+   * {
+   *   "elapsed":0,
+   *   "code":403,
+   *   "original_string":"1469539185.270      0 139.196.181.68 TCP_DENIED/403 3617 CONNECT
search.yahoo.com:443 - NONE/- text/html",
+   *   "method":"CONNECT",
+   *   "bytes":3617,
+   *   "action":"TCP_DENIED",
+   *   "url":"search.yahoo.com:443",
+   *   "ip_src_addr":"139.196.181.68",
+   *   "timestamp":1469539185270,
+   *   "ip_dst_addr": null
+   * }
+   */
+  @Multiline
+  private String result2;
 
   @Override
-  public String getExpectedParsedString() {
-    return expectedParsedString;
+  public Map<String,String> getTestData() {
+
+    String input1 = "1461576382.642    161 127.0.0.1 TCP_MISS/200 103701 GET http://www.cnn.com/
- DIRECT/199.27.79.73 text/html";
+    String input2 = "1469539185.270      0 139.196.181.68 TCP_DENIED/403 3617 CONNECT search.yahoo.com:443
- NONE/- text/html";
+
+    HashMap testData = new HashMap<String,String>();
+    testData.put(input1,result1);
+    testData.put(input2,result2);
+    return testData;
+
   }
 
+
   @Override
   public String getGrokPath() {
     return "../metron-parsers/src/main/resources/patterns/squid";

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/bbd6474e/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/YafParserTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/YafParserTest.java
b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/YafParserTest.java
index 1c1da4d..8dd75a0 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/YafParserTest.java
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/YafParserTest.java
@@ -20,20 +20,12 @@ package org.apache.metron.parsers;
 import org.adrianwalker.multilinestring.Multiline;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 public class YafParserTest extends GrokParserTest {
 
-  @Override
-  public String getRawMessage() {
-    return "2016-01-28 15:29:48.512|2016-01-28 15:29:48.512|   0.000|   0.000|  6|      
                   216.21.170.221|   80|                               10.0.2.15|39468|  
   AS|       0|       0|       0|22efa001|00000000|000|000|       1|      44|       0|   
   0|    0|idle";
-  }
-
-  @Override
-  public String getGrokPath() {
-    return "../metron-parsers/src/main/resources/patterns/yaf";
-  }
-
   /**
    * {
    "iflags": "AS",
@@ -64,13 +56,25 @@ public class YafParserTest extends GrokParserTest {
    }
    */
   @Multiline
-  public String expectedParsedString;
+  public String result;
 
   @Override
-  public String getExpectedParsedString() {
-    return expectedParsedString;
+  public Map getTestData() {
+
+    Map testData = new HashMap<String,String>();
+    String input = "2016-01-28 15:29:48.512|2016-01-28 15:29:48.512|   0.000|   0.000|  6|
                         216.21.170.221|   80|                               10.0.2.15|39468|
     AS|       0|       0|       0|22efa001|00000000|000|000|       1|      44|       0| 
     0|    0|idle";
+    testData.put(input,result);
+    return testData;
+
+  }
+
+  @Override
+  public String getGrokPath() {
+    return "../metron-parsers/src/main/resources/patterns/yaf";
   }
 
+
+
   @Override
   public String getGrokPatternLabel() {
     return "YAF_DELIMITED";


Mime
View raw message