metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ceste...@apache.org
Subject metron git commit: METRON-996 Performance improvement for ASA parser closes apache/incubator-metron#617
Date Mon, 19 Jun 2017 15:31:13 GMT
Repository: metron
Updated Branches:
  refs/heads/master de2c871ab -> b76bcd5f2


METRON-996 Performance improvement for ASA parser closes apache/incubator-metron#617


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/b76bcd5f
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/b76bcd5f
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/b76bcd5f

Branch: refs/heads/master
Commit: b76bcd5f2405db122c2bf9d19128f9efb20b8458
Parents: de2c871
Author: Simon Elliston Ball <simon@simonellistonball.com>
Authored: Mon Jun 19 11:30:58 2017 -0400
Committer: cstella <cestella@gmail.com>
Committed: Mon Jun 19 11:30:58 2017 -0400

----------------------------------------------------------------------
 .../metron/parsers/asa/BasicAsaParser.java      | 366 ++++++++++---------
 1 file changed, 191 insertions(+), 175 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/metron/blob/b76bcd5f/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/asa/BasicAsaParser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/asa/BasicAsaParser.java
b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/asa/BasicAsaParser.java
index ad1db83..daac141 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/asa/BasicAsaParser.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/asa/BasicAsaParser.java
@@ -33,187 +33,203 @@ import java.io.*;
 import java.time.Clock;
 import java.time.ZoneId;
 import java.util.*;
+import java.util.Map.Entry;
 
 public class BasicAsaParser extends BasicParser {
 
-    protected static final Logger LOG = LoggerFactory.getLogger(BasicAsaParser.class);
-
-    private Grok asaGrok;
-    protected Clock deviceClock;
-
-    private static final Map<String, String> patternMap = ImmutableMap.<String,
String>builder()
-            .put("ASA-2-106001", "CISCOFW106001")
-		    .put("ASA-2-106006", "CISCOFW106006_106007_106010")
-		    .put("ASA-2-106007", "CISCOFW106006_106007_106010")
-		    .put("ASA-2-106010", "CISCOFW106006_106007_106010")
-		    .put("ASA-3-106014", "CISCOFW106014")
-		    .put("ASA-6-106015", "CISCOFW106015")
-		    .put("ASA-1-106021", "CISCOFW106021")
-		    .put("ASA-4-106023", "CISCOFW106023")
-		    .put("ASA-5-106100", "CISCOFW106100")
-		    .put("ASA-6-110002", "CISCOFW110002")
-		    .put("ASA-6-302010", "CISCOFW302010")
-		    .put("ASA-6-302013", "CISCOFW302013_302014_302015_302016")
-		    .put("ASA-6-302014", "CISCOFW302013_302014_302015_302016")
-		    .put("ASA-6-302015", "CISCOFW302013_302014_302015_302016")
-		    .put("ASA-6-302016", "CISCOFW302013_302014_302015_302016")
-		    .put("ASA-6-302020", "CISCOFW302020_302021")
-		    .put("ASA-6-302021", "CISCOFW302020_302021")
-		    .put("ASA-6-305011", "CISCOFW305011")
-		    .put("ASA-3-313001", "CISCOFW313001_313004_313008")
-		    .put("ASA-3-313004", "CISCOFW313001_313004_313008")
-		    .put("ASA-3-313008", "CISCOFW313001_313004_313008")
-		    .put("ASA-4-313005", "CISCOFW313005")
-		    .put("ASA-4-402117", "CISCOFW402117")
-		    .put("ASA-4-402119", "CISCOFW402119")
-		    .put("ASA-4-419001", "CISCOFW419001")
-		    .put("ASA-4-419002", "CISCOFW419002")
-		    .put("ASA-4-500004", "CISCOFW500004")
-		    .put("ASA-6-602303", "CISCOFW602303_602304")
-		    .put("ASA-6-602304", "CISCOFW602303_602304")
-		    .put("ASA-7-710001", "CISCOFW710001_710002_710003_710005_710006")
-		    .put("ASA-7-710002", "CISCOFW710001_710002_710003_710005_710006")
-		    .put("ASA-7-710003", "CISCOFW710001_710002_710003_710005_710006")
-		    .put("ASA-7-710005", "CISCOFW710001_710002_710003_710005_710006")
-		    .put("ASA-7-710006", "CISCOFW710001_710002_710003_710005_710006")
-		    .put("ASA-6-713172", "CISCOFW713172")
-		    .put("ASA-4-733100", "CISCOFW733100")
-		    .put("ASA-6-305012", "CISCOFW305012")
-		    .put("ASA-7-609001", "CISCOFW609001")
-		    .put("ASA-7-609002", "CISCOFW609002")
-            .put("ASA-5-713041", "CISCOFW713041")
-            .build();
-
-    @Override
-    public void configure(Map<String, Object> parserConfig) {
-        String timeZone = (String) parserConfig.get("deviceTimeZone");
-        if (timeZone != null)
-            deviceClock = Clock.system(ZoneId.of(timeZone));
-        else {
-            deviceClock = Clock.systemUTC();
-            LOG.warn("[Metron] No device time zone provided; defaulting to UTC");
-        }
+  protected static final Logger LOG = LoggerFactory.getLogger(BasicAsaParser.class);
+
+  protected Clock deviceClock;
+  private String syslogPattern = "%{CISCO_TAGGED_SYSLOG}";
+
+  private Grok syslogGrok;
+
+  private static final Map<String, String> patternMap = ImmutableMap.<String, String>
builder()
+      .put("ASA-2-106001", "CISCOFW106001")
+      .put("ASA-2-106006", "CISCOFW106006_106007_106010")
+      .put("ASA-2-106007", "CISCOFW106006_106007_106010")
+      .put("ASA-2-106010", "CISCOFW106006_106007_106010")
+      .put("ASA-3-106014", "CISCOFW106014")
+      .put("ASA-6-106015", "CISCOFW106015")
+      .put("ASA-1-106021", "CISCOFW106021")
+      .put("ASA-4-106023", "CISCOFW106023")
+      .put("ASA-5-106100", "CISCOFW106100")
+      .put("ASA-6-110002", "CISCOFW110002")
+      .put("ASA-6-302010", "CISCOFW302010")
+      .put("ASA-6-302013", "CISCOFW302013_302014_302015_302016")
+      .put("ASA-6-302014", "CISCOFW302013_302014_302015_302016")
+      .put("ASA-6-302015", "CISCOFW302013_302014_302015_302016")
+      .put("ASA-6-302016", "CISCOFW302013_302014_302015_302016")
+      .put("ASA-6-302020", "CISCOFW302020_302021")
+      .put("ASA-6-302021", "CISCOFW302020_302021")
+      .put("ASA-6-305011", "CISCOFW305011")
+      .put("ASA-3-313001", "CISCOFW313001_313004_313008")
+      .put("ASA-3-313004", "CISCOFW313001_313004_313008")
+      .put("ASA-3-313008", "CISCOFW313001_313004_313008")
+      .put("ASA-4-313005", "CISCOFW313005")
+      .put("ASA-4-402117", "CISCOFW402117")
+      .put("ASA-4-402119", "CISCOFW402119")
+      .put("ASA-4-419001", "CISCOFW419001")
+      .put("ASA-4-419002", "CISCOFW419002")
+      .put("ASA-4-500004", "CISCOFW500004")
+      .put("ASA-6-602303", "CISCOFW602303_602304")
+      .put("ASA-6-602304", "CISCOFW602303_602304")
+      .put("ASA-7-710001", "CISCOFW710001_710002_710003_710005_710006")
+      .put("ASA-7-710002", "CISCOFW710001_710002_710003_710005_710006")
+      .put("ASA-7-710003", "CISCOFW710001_710002_710003_710005_710006")
+      .put("ASA-7-710005", "CISCOFW710001_710002_710003_710005_710006")
+      .put("ASA-7-710006", "CISCOFW710001_710002_710003_710005_710006")
+      .put("ASA-6-713172", "CISCOFW713172")
+      .put("ASA-4-733100", "CISCOFW733100")
+      .put("ASA-6-305012", "CISCOFW305012")
+      .put("ASA-7-609001", "CISCOFW609001")
+      .put("ASA-7-609002", "CISCOFW609002")
+      .put("ASA-5-713041", "CISCOFW713041")
+      .build();
+
+  private Map<String, Grok> grokers = new HashMap<String, Grok>(patternMap.size());
+
+  @Override
+  public void configure(Map<String, Object> parserConfig) {
+    String timeZone = (String) parserConfig.get("deviceTimeZone");
+    if (timeZone != null)
+      deviceClock = Clock.system(ZoneId.of(timeZone));
+    else {
+      deviceClock = Clock.systemUTC();
+      LOG.warn("[Metron] No device time zone provided; defaulting to UTC");
+    }
+  }
+
+  private void addGrok(String key, String pattern) throws GrokException {
+    Grok grok = new Grok();
+    InputStream patternStream = this.getClass().getResourceAsStream("/patterns/asa");
+    grok.addPatternFromReader(new InputStreamReader(patternStream));
+    grok.compile("%{" + pattern + "}");
+    grokers.put(key, grok);
+  }
+
+  @Override
+  public void init() {
+    syslogGrok = new Grok();
+    InputStream syslogStream = this.getClass().getResourceAsStream("/patterns/asa");
+    try {
+      syslogGrok.addPatternFromReader(new InputStreamReader(syslogStream));
+      syslogGrok.compile(syslogPattern);
+    } catch (GrokException e) {
+      LOG.error("[Metron] Failed to load grok patterns from jar", e);
+      throw new RuntimeException(e.getMessage(), e);
+    }
+
+    for (Entry<String, String> pattern : patternMap.entrySet()) {
+      try {
+	addGrok(pattern.getKey(), pattern.getValue());
+      } catch (GrokException e) {
+	LOG.error("[Metron] Failed to load grok pattern %s for ASA tag %s", pattern.getValue(),
pattern.getKey());
+      }
     }
 
-    @Override
-    public void init() {
-        asaGrok = new Grok();
-        InputStream patternStream = this.getClass().getResourceAsStream("/patterns/asa");
-        try {
-            asaGrok.addPatternFromReader(new InputStreamReader(patternStream));
-        } catch (GrokException e) {
-            LOG.error("[Metron] Failed to load grok patterns from jar", e);
-            throw new RuntimeException(e.getMessage(), e);
-        }
-        LOG.info("[Metron] CISCO ASA Parser Initialized");
+    LOG.info("[Metron] CISCO ASA Parser Initialized");
+  }
+
+  @Override
+  public List<JSONObject> parse(byte[] rawMessage) {
+    String logLine = "";
+    String messagePattern = "";
+    JSONObject metronJson = new JSONObject();
+    List<JSONObject> messages = new ArrayList<>();
+    Map<String, Object> syslogJson = new HashMap<String, Object>();
+
+    try {
+      logLine = new String(rawMessage, "UTF-8");
+    } catch (UnsupportedEncodingException e) {
+      LOG.error("[Metron] Could not read raw message", e);
+      throw new RuntimeException(e.getMessage(), e);
     }
 
-    @Override
-    public List<JSONObject> parse(byte[] rawMessage) {
-        String logLine = "";
-        String syslogPattern = "%{CISCO_TAGGED_SYSLOG}";
-        String messagePattern = "";
-        JSONObject metronJson = new JSONObject();
-        List<JSONObject> messages = new ArrayList<>();
-        Map<String, Object> syslogJson = new HashMap<String, Object>();
-
-        try {
-            logLine = new String(rawMessage, "UTF-8");
-        } catch (UnsupportedEncodingException e) {
-            LOG.error("[Metron] Could not read raw message", e);
-            throw new RuntimeException(e.getMessage(), e);
-        }
-
-        try {
-            LOG.debug("[Metron] Started parsing raw message: {}", logLine);
-
-            asaGrok.compile(syslogPattern);
-            Match syslogMatch = asaGrok.match(logLine);
-            syslogMatch.captures();
-            if(!syslogMatch.isNull()) {
-                syslogJson = syslogMatch.toMap();
-                LOG.trace("[Metron] Grok CISCO ASA syslog matches: {}", syslogMatch.toJson());
-
-                metronJson.put(Constants.Fields.ORIGINAL.getName(), logLine);
-                metronJson.put(Constants.Fields.TIMESTAMP.getName(),
-                        SyslogUtils.parseTimestampToEpochMillis((String) syslogJson.get("CISCOTIMESTAMP"),
deviceClock));
-                metronJson.put("ciscotag", syslogJson.get("CISCOTAG"));
-                metronJson.put("syslog_severity", SyslogUtils.getSeverityFromPriority((int)
syslogJson.get("syslog_pri")));
-                metronJson.put("syslog_facility", SyslogUtils.getFacilityFromPriority((int)
syslogJson.get("syslog_pri")));
-                
-                
-                if (syslogJson.get("syslog_host")!=null) { 
-                	metronJson.put("syslog_host", syslogJson.get("syslog_host")); 
-            	}
-                if (syslogJson.get("syslog_prog")!=null) { 
-                    metronJson.put("syslog_prog", syslogJson.get("syslog_prog"));
-                }
-                
-            }
-            else
-                throw new RuntimeException(String.format("[Metron] Message '%s' does not
match pattern '%s'", logLine, syslogPattern));
-        } catch (GrokException e) {
-            LOG.error(String.format("[Metron] Could not compile grok pattern '%s'", syslogPattern),
e);
-            throw new RuntimeException(e.getMessage(), e);
-        } catch (ParseException e) {
-            LOG.error("[Metron] Could not parse message timestamp", e);
-            throw new RuntimeException(e.getMessage(), e);
-        } catch (RuntimeException e) {
-            LOG.error(e.getMessage(), e);
-            throw new RuntimeException(e.getMessage(), e);
-        }
-
-        try {
-            messagePattern = patternMap.get(syslogJson.get("CISCOTAG"));
-            if (messagePattern == null)
-                LOG.info("[Metron] No pattern for ciscotag '{}'", syslogJson.get("CISCOTAG"));
-            else {
-                asaGrok.compile("%{" + messagePattern + "}");
-                Match messageMatch = asaGrok.match((String) syslogJson.get("message"));
-                messageMatch.captures();
-                if (!messageMatch.isNull()) {
-                    Map<String, Object> messageJson = messageMatch.toMap();
-                    LOG.trace("[Metron] Grok CISCO ASA message matches: {}", messageMatch.toJson());
-
-                    String src_ip = (String) messageJson.get("src_ip");
-                    if (src_ip != null)
-                        metronJson.put(Constants.Fields.SRC_ADDR.getName(), src_ip);
-
-                    Integer src_port = (Integer) messageJson.get("src_port");
-                    if (src_port != null)
-                        metronJson.put(Constants.Fields.SRC_PORT.getName(), src_port);
-
-                    String dst_ip = (String) messageJson.get("dst_ip");
-                    if (dst_ip != null)
-                        metronJson.put(Constants.Fields.DST_ADDR.getName(), dst_ip);
-
-                    Integer dst_port = (Integer) messageJson.get("dst_port");
-                    if (dst_port != null)
-                        metronJson.put(Constants.Fields.DST_PORT.getName(), dst_port);
-
-                    String protocol = (String) messageJson.get("protocol");
-                    if (protocol != null)
-                        metronJson.put(Constants.Fields.PROTOCOL.getName(), protocol.toLowerCase());
-
-                    String action = (String) messageJson.get("action");
-                    if (action != null)
-                        metronJson.put("action", action.toLowerCase());
-                }
-                else
-                    LOG.warn("[Metron] Message '{}' did not match pattern for ciscotag '{}'",
logLine, syslogJson.get("CISCOTAG"));
-            }
-
-            LOG.debug("[Metron] Final normalized message: {}", metronJson.toString());
-
-        } catch (GrokException e) {
-            LOG.error(String.format("[Metron] Could not compile grok pattern '%s'", messagePattern),
e);
-            throw new RuntimeException(e.getMessage(), e);
-        } catch (RuntimeException e) {
-            LOG.error(e.getMessage(), e);
-            throw new RuntimeException(e.getMessage(), e);
-        }
-
-        messages.add(metronJson);
-        return messages;
+    try {
+      LOG.debug("[Metron] Started parsing raw message: {}", logLine);
+      Match syslogMatch = syslogGrok.match(logLine);
+      syslogMatch.captures();
+      if (!syslogMatch.isNull()) {
+	syslogJson = syslogMatch.toMap();
+	LOG.trace("[Metron] Grok CISCO ASA syslog matches: {}", syslogMatch.toJson());
+
+	metronJson.put(Constants.Fields.ORIGINAL.getName(), logLine);
+	metronJson.put(Constants.Fields.TIMESTAMP.getName(),
+	    SyslogUtils.parseTimestampToEpochMillis((String) syslogJson.get("CISCOTIMESTAMP"), deviceClock));
+	metronJson.put("ciscotag", syslogJson.get("CISCOTAG"));
+	metronJson.put("syslog_severity", SyslogUtils.getSeverityFromPriority((int) syslogJson.get("syslog_pri")));
+	metronJson.put("syslog_facility", SyslogUtils.getFacilityFromPriority((int) syslogJson.get("syslog_pri")));
+
+	if (syslogJson.get("syslog_host") != null) {
+	  metronJson.put("syslog_host", syslogJson.get("syslog_host"));
+	}
+	if (syslogJson.get("syslog_prog") != null) {
+	  metronJson.put("syslog_prog", syslogJson.get("syslog_prog"));
+	}
+
+      } else
+	throw new RuntimeException(
+	    String.format("[Metron] Message '%s' does not match pattern '%s'", logLine, syslogPattern));
+    } catch (ParseException e) {
+      LOG.error("[Metron] Could not parse message timestamp", e);
+      throw new RuntimeException(e.getMessage(), e);
+    } catch (RuntimeException e) {
+      LOG.error(e.getMessage(), e);
+      throw new RuntimeException(e.getMessage(), e);
     }
+
+    try {
+      messagePattern = (String) syslogJson.get("CISCOTAG");
+      Grok asaGrok = grokers.get(messagePattern);
+
+      if (asaGrok == null)
+	LOG.info("[Metron] No pattern for ciscotag '{}'", syslogJson.get("CISCOTAG"));
+      else {
+
+	String messageContent = (String) syslogJson.get("message");
+	Match messageMatch = asaGrok.match(messageContent);
+	messageMatch.captures();
+	if (!messageMatch.isNull()) {
+	  Map<String, Object> messageJson = messageMatch.toMap();
+	  LOG.trace("[Metron] Grok CISCO ASA message matches: {}", messageMatch.toJson());
+
+	  String src_ip = (String) messageJson.get("src_ip");
+	  if (src_ip != null)
+	    metronJson.put(Constants.Fields.SRC_ADDR.getName(), src_ip);
+
+	  Integer src_port = (Integer) messageJson.get("src_port");
+	  if (src_port != null)
+	    metronJson.put(Constants.Fields.SRC_PORT.getName(), src_port);
+
+	  String dst_ip = (String) messageJson.get("dst_ip");
+	  if (dst_ip != null)
+	    metronJson.put(Constants.Fields.DST_ADDR.getName(), dst_ip);
+
+	  Integer dst_port = (Integer) messageJson.get("dst_port");
+	  if (dst_port != null)
+	    metronJson.put(Constants.Fields.DST_PORT.getName(), dst_port);
+
+	  String protocol = (String) messageJson.get("protocol");
+	  if (protocol != null)
+	    metronJson.put(Constants.Fields.PROTOCOL.getName(), protocol.toLowerCase());
+
+	  String action = (String) messageJson.get("action");
+	  if (action != null)
+	    metronJson.put("action", action.toLowerCase());
+	} else
+	  LOG.warn("[Metron] Message '{}' did not match pattern for ciscotag '{}'", logLine,
+	      syslogJson.get("CISCOTAG"));
+      }
+
+      LOG.debug("[Metron] Final normalized message: {}", metronJson.toString());
+
+    } catch (RuntimeException e) {
+      LOG.error(e.getMessage(), e);
+      throw new RuntimeException(e.getMessage(), e);
+    }
+
+    messages.add(metronJson);
+    return messages;
+  }
 }


Mime
View raw message