flume-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From roshann...@apache.org
Subject flume git commit: FLUME-2889: Fixes to DateTime computations
Date Mon, 25 Apr 2016 20:19:13 GMT
Repository: flume
Updated Branches:
  refs/heads/flume-1.7 8d06a72d9 -> 49a370a62


FLUME-2889: Fixes to DateTime computations

(Tristan Stevens via Roshan Naik)


Project: http://git-wip-us.apache.org/repos/asf/flume/repo
Commit: http://git-wip-us.apache.org/repos/asf/flume/commit/49a370a6
Tree: http://git-wip-us.apache.org/repos/asf/flume/tree/49a370a6
Diff: http://git-wip-us.apache.org/repos/asf/flume/diff/49a370a6

Branch: refs/heads/flume-1.7
Commit: 49a370a62450b654aed52ef680b7c39e22a74948
Parents: 8d06a72
Author: Roshan Naik <roshan@hortonworks.com>
Authored: Mon Apr 25 12:43:31 2016 -0700
Committer: Roshan Naik <roshan@hortonworks.com>
Committed: Mon Apr 25 12:43:31 2016 -0700

----------------------------------------------------------------------
 .../org/apache/flume/source/SyslogParser.java   | 11 +++-
 .../org/apache/flume/source/SyslogUtils.java    | 40 ++++++++++++-
 .../apache/flume/source/TestSyslogUtils.java    | 62 ++++++++++++++++++--
 3 files changed, 104 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flume/blob/49a370a6/flume-ng-core/src/main/java/org/apache/flume/source/SyslogParser.java
----------------------------------------------------------------------
diff --git a/flume-ng-core/src/main/java/org/apache/flume/source/SyslogParser.java b/flume-ng-core/src/main/java/org/apache/flume/source/SyslogParser.java
index c8245ff..b57ffac 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/source/SyslogParser.java
+++ b/flume-ng-core/src/main/java/org/apache/flume/source/SyslogParser.java
@@ -317,9 +317,16 @@ public class SyslogParser {
       return 0;
     }
 
-    // try to deal with boundary cases, i.e. new year's eve.
     // rfc3164 dates are really dumb.
-    // NB: cannot handle replaying of old logs or going back to the future
+    /*
+     * Some code to try and add some smarts to the year insertion as without a year in the
message we
+     * need to make some educated guessing.
+     * First set the "fixed" to be the timestamp with the current year.
+     * If the "fixed" time is more than one month in the future then roll it back a year.
+     * If the "fixed" time is more than eleven months in the past then roll it forward a
year.
+     * This gives us a 12 month rolling window (11 months in the past, 1 month in the future)
of timestamps.
+     */
+
     if (date != null) {
       DateTime fixed = date.withYear(year);
 

http://git-wip-us.apache.org/repos/asf/flume/blob/49a370a6/flume-ng-core/src/main/java/org/apache/flume/source/SyslogUtils.java
----------------------------------------------------------------------
diff --git a/flume-ng-core/src/main/java/org/apache/flume/source/SyslogUtils.java b/flume-ng-core/src/main/java/org/apache/flume/source/SyslogUtils.java
index 5a9f4c8..4866183 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/source/SyslogUtils.java
+++ b/flume-ng-core/src/main/java/org/apache/flume/source/SyslogUtils.java
@@ -33,6 +33,7 @@ import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Calendar;
+import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -369,7 +370,44 @@ public class SyslogUtils {
             // try the available time formats to timestamp
             for (int dt = 0; dt < fmt.dateFormat.size(); dt++) {
               try {
-                timeStamp = String.valueOf(fmt.dateFormat.get(dt).parse(value).getTime());
+                Date parsedDate = fmt.dateFormat.get(dt).parse(value);
+                /*
+                 * Some code to try and add some smarts to the year insertion.
+                 * Original code just added the current year which was okay-ish, but around
January 1st becomes
+                 * pretty naïve.
+                 * The current year is added above. This code, if the year has been added
does the following:
+                 * 1. Compute what the computed time, but one month in the past would be.
+                 * 2. Compute what the computed time, but eleven months in the future would
be.
+                 * If the computed time is more than one month in the future then roll it
back a year.
+                 * If the computed time is more than eleven months in the past then roll
it forward a year.
+                 * This gives us a 12 month rolling window (11 months in the past, 1 month
in the future) of timestamps.
+                 */
+                if (fmt.addYear) {
+                  Calendar cal = Calendar.getInstance();
+                  cal.setTime(parsedDate);
+                  Calendar calMinusOneMonth = Calendar.getInstance();
+                  calMinusOneMonth.setTime(parsedDate);
+                  calMinusOneMonth.add(Calendar.MONTH, -1);
+
+                  Calendar calPlusElevenMonths = Calendar.getInstance();
+                  calPlusElevenMonths.setTime(parsedDate);
+                  calPlusElevenMonths.add(Calendar.MONTH, +11);
+
+                  if (cal.getTimeInMillis() > System.currentTimeMillis() && calMinusOneMonth.getTimeInMillis()
> System.currentTimeMillis()) {
+                    //Need to roll back a year
+                    Calendar c1 = Calendar.getInstance();
+                    c1.setTime(parsedDate);
+                    c1.add(Calendar.YEAR, -1);
+                    parsedDate = c1.getTime();
+                  } else if (cal.getTimeInMillis() < System.currentTimeMillis() &&
calPlusElevenMonths.getTimeInMillis() < System.currentTimeMillis() ) {
+                    //Need to roll forward a year
+                    Calendar c1 = Calendar.getInstance();
+                    c1.setTime(parsedDate);
+                    c1.add(Calendar.YEAR, -1);
+                    parsedDate = c1.getTime();
+                  }
+                }
+                timeStamp = String.valueOf(parsedDate.getTime());
                 break; // done. formatted the time
               } catch (ParseException e) {
                 // Error formatting the timeStamp, try next format

http://git-wip-us.apache.org/repos/asf/flume/blob/49a370a6/flume-ng-core/src/test/java/org/apache/flume/source/TestSyslogUtils.java
----------------------------------------------------------------------
diff --git a/flume-ng-core/src/test/java/org/apache/flume/source/TestSyslogUtils.java b/flume-ng-core/src/test/java/org/apache/flume/source/TestSyslogUtils.java
index be4598e..1c005ff 100644
--- a/flume-ng-core/src/test/java/org/apache/flume/source/TestSyslogUtils.java
+++ b/flume-ng-core/src/test/java/org/apache/flume/source/TestSyslogUtils.java
@@ -28,6 +28,7 @@ import org.junit.Test;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.Calendar;
+import java.util.Date;
 import java.util.HashSet;
 import java.util.Locale;
 import java.util.Map;
@@ -130,25 +131,33 @@ public class TestSyslogUtils {
 
   @Test
   public void TestHeader9() throws ParseException {
-    String stamp1 = "Apr 11 13:14:04";
+    SimpleDateFormat sdf = new SimpleDateFormat("MMM  d hh:MM:ss");
+    Calendar cal = Calendar.getInstance();
+
+    String year = String.valueOf(cal.get(Calendar.YEAR));
+    String stamp1 = sdf.format(cal.getTime());
     String format1 = "yyyyMMM d HH:mm:ss";
     String host1 = "ubuntu-11.cloudera.com";
     String data1 = "some msg";
     // timestamp with 'Z' appended, translates to UTC
     String msg1 = "<10>" + stamp1 + " " + host1 + " " + data1 + "\n";
-    checkHeader(msg1, String.valueOf(Calendar.getInstance().get(Calendar.YEAR)) + stamp1,
+    checkHeader(msg1, year + stamp1,
         format1, host1, data1);
   }
 
   @Test
   public void TestHeader10() throws ParseException {
-    String stamp1 = "Apr  1 13:14:04";
+    SimpleDateFormat sdf = new SimpleDateFormat("MMM  d hh:MM:ss");
+    Calendar cal = Calendar.getInstance();
+
+    String year = String.valueOf(cal.get(Calendar.YEAR));
+    String stamp1 = sdf.format(cal.getTime());
     String format1 = "yyyyMMM d HH:mm:ss";
     String host1 = "ubuntu-11.cloudera.com";
     String data1 = "some msg";
     // timestamp with 'Z' appended, translates to UTC
     String msg1 = "<10>" + stamp1 + " " + host1 + " " + data1 + "\n";
-    checkHeader(msg1, String.valueOf(Calendar.getInstance().get(Calendar.YEAR)) + stamp1,
+    checkHeader(msg1, year + stamp1,
         format1, host1, data1);
   }
 
@@ -169,15 +178,56 @@ public class TestSyslogUtils {
 
   @Test
   public void TestRfc3164HeaderApacheLogWithNulls() throws ParseException {
-    String stamp1 = "Apr  1 13:14:04";
+    SimpleDateFormat sdf = new SimpleDateFormat("MMM  d hh:MM:ss");
+    Calendar cal = Calendar.getInstance();
+
+    String year = String.valueOf(cal.get(Calendar.YEAR));
+    String stamp1 = sdf.format(cal.getTime());
     String format1 = "yyyyMMM d HH:mm:ss";
     String host1 = "ubuntu-11.cloudera.com";
     String data1 = "- hyphen_null_breaks_5424_pattern [07/Jun/2012:14:46:44 -0600]";
     String msg1 = "<10>" + stamp1 + " " + host1 + " " + data1 + "\n";
-    checkHeader(msg1, String.valueOf(Calendar.getInstance().get(Calendar.YEAR)) + stamp1,
+    checkHeader(msg1, year + stamp1,
             format1, host1, data1);
   }
 
+  @Test
+  public void TestRfc3164Dates() throws ParseException {
+    /*
+     * This test creates a series of dates that range from 10 months in the past to (5 days
short of)
+     * one month in the future. This tests that the year addition code is clever enough to
handle scenarios
+     * where the event received was generated in a different year to what flume considers
to be "current"
+     * (e.g. where there has been some lag somewhere, especially when flicking over on New
Year's eve, or
+     * when you are about to flick over and the flume's system clock is slightly slower than
the Syslog
+     * source's clock).
+     */
+    for (int i=-10; i<=1; i++) {
+      SimpleDateFormat sdf = new SimpleDateFormat("MMM  d hh:MM:ss");
+      Date date = new Date(System.currentTimeMillis());
+      Calendar cal = Calendar.getInstance();
+      cal.setTime(date);
+      cal.add(Calendar.MONTH, i);
+
+      //Small tweak to avoid the 1 month in the future ticking over by a few seconds between
now
+      //and when the checkHeader actually runs
+      if (i==1) {
+        cal.add(Calendar.DAY_OF_MONTH, -1);
+      }
+
+      String stamp1 = sdf.format(cal.getTime());
+
+      String year = String.valueOf(cal.get(Calendar.YEAR));
+      String format1 = "yyyyMMM d HH:mm:ss";
+      String host1 = "ubuntu-11.cloudera.com";
+      String data1 = "some msg";
+
+      // timestamp with 'Z' appended, translates to UTC
+      String msg1 = "<10>" + stamp1 + " " + host1 + " " + data1 + "\n";
+      checkHeader(msg1, year + stamp1,
+          format1, host1, data1);
+    }
+  }
+
   public static void checkHeader(String keepFields, String msg1, String stamp1, String format1,
                                  String host1, String data1) throws ParseException {
     SyslogUtils util;


Mime
View raw message