falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ajayyad...@apache.org
Subject falcon git commit: FALCON-1372 Retention does not work in corner cases. Contributed by Balu Vellanki.
Date Sun, 15 Nov 2015 20:59:17 GMT
Repository: falcon
Updated Branches:
  refs/heads/master ca7a91232 -> 7285a4487


FALCON-1372 Retention does not work in corner cases. Contributed by Balu Vellanki.


Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/7285a448
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/7285a448
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/7285a448

Branch: refs/heads/master
Commit: 7285a44875b6dae67610ff7a65fb6290c5aefbf5
Parents: ca7a912
Author: Ajay Yadav <ajay.yadav@inmobi.com>
Authored: Mon Nov 16 01:36:30 2015 +0530
Committer: Ajay Yadav <ajay.yadav@inmobi.com>
Committed: Mon Nov 16 01:36:30 2015 +0530

----------------------------------------------------------------------
 CHANGES.txt                                     |  2 +
 .../org/apache/falcon/entity/FeedHelper.java    | 10 ++++-
 .../falcon/entity/common/FeedDataPath.java      |  8 ++--
 .../apache/falcon/entity/FeedHelperTest.java    | 43 ++++++++++++--------
 4 files changed, 43 insertions(+), 20 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/falcon/blob/7285a448/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 7ae9b0f..59d8f19 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -29,6 +29,8 @@ Trunk (Unreleased)
   OPTIMIZATIONS
 
   BUG FIXES
+    FALCON-1372 Retention does not work in corner cases(Balu Vellanki via Ajay Yadava)
+
     FALCON-1569 Bug in setting the frequency of Feed retention coordinator(Sowmya Ramesh)
 
     FALCON-1578 post-processing action fails with javax.servlet.jsp.el.ELException(Venkat
Ramachandran via Ajay Yadava)

http://git-wip-us.apache.org/repos/asf/falcon/blob/7285a448/common/src/main/java/org/apache/falcon/entity/FeedHelper.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/FeedHelper.java b/common/src/main/java/org/apache/falcon/entity/FeedHelper.java
index 8c55e41..d601c5d 100644
--- a/common/src/main/java/org/apache/falcon/entity/FeedHelper.java
+++ b/common/src/main/java/org/apache/falcon/entity/FeedHelper.java
@@ -475,8 +475,16 @@ public final class FeedHelper {
         //Reset other fields
         for (FeedDataPath.VARS var : FeedDataPath.VARS.values()) {
             if (!matchedVars.contains(var)) {
-                cal.set(var.getCalendarField(), 0);
+                switch (var.getCalendarField()) {
+                case Calendar.DAY_OF_MONTH:
+                    cal.set(var.getCalendarField(), 1);
+                    break;
+                default:
+                    cal.set(var.getCalendarField(), 0);
+                }
             }
+            cal.set(Calendar.SECOND, 0);
+            cal.set(Calendar.MILLISECOND, 0);
         }
         return cal.getTime();
     }

http://git-wip-us.apache.org/repos/asf/falcon/blob/7285a448/common/src/main/java/org/apache/falcon/entity/common/FeedDataPath.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/common/FeedDataPath.java b/common/src/main/java/org/apache/falcon/entity/common/FeedDataPath.java
index afe913d..51568fb 100644
--- a/common/src/main/java/org/apache/falcon/entity/common/FeedDataPath.java
+++ b/common/src/main/java/org/apache/falcon/entity/common/FeedDataPath.java
@@ -30,10 +30,12 @@ public final class FeedDataPath {
     /**
      * Standard variables for feed time components.
      */
-    public static enum VARS {
-        YEAR("([0-9]{4})", Calendar.YEAR, 4), MONTH("(0[1-9]|1[0-2])", Calendar.MONTH, 2),
+    public enum VARS {
+        YEAR("([0-9]{4})", Calendar.YEAR, 4),
+        MONTH("(0[1-9]|1[0-2])", Calendar.MONTH, 2),
         DAY("(0[1-9]|1[0-9]|2[0-9]|3[0-1])", Calendar.DAY_OF_MONTH, 2),
-        HOUR("([0-1][0-9]|2[0-4])", Calendar.HOUR_OF_DAY, 2), MINUTE("([0-5][0-9]|60)", Calendar.MINUTE,
2);
+        HOUR("([0-1][0-9]|2[0-4])", Calendar.HOUR_OF_DAY, 2),
+        MINUTE("([0-5][0-9]|60)", Calendar.MINUTE, 2);
 
         private final Pattern pattern;
         private final String valuePattern;

http://git-wip-us.apache.org/repos/asf/falcon/blob/7285a448/common/src/test/java/org/apache/falcon/entity/FeedHelperTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/falcon/entity/FeedHelperTest.java b/common/src/test/java/org/apache/falcon/entity/FeedHelperTest.java
index 10dac49..4094556 100644
--- a/common/src/test/java/org/apache/falcon/entity/FeedHelperTest.java
+++ b/common/src/test/java/org/apache/falcon/entity/FeedHelperTest.java
@@ -30,23 +30,22 @@ import org.apache.falcon.entity.v0.cluster.Properties;
 import org.apache.falcon.entity.v0.cluster.Property;
 import org.apache.falcon.entity.v0.feed.Argument;
 import org.apache.falcon.entity.v0.feed.Arguments;
-import org.apache.falcon.entity.v0.feed.ExtractMethod;
-import org.apache.falcon.entity.v0.feed.Clusters;
 import org.apache.falcon.entity.v0.feed.ClusterType;
+import org.apache.falcon.entity.v0.feed.Clusters;
 import org.apache.falcon.entity.v0.feed.Extract;
+import org.apache.falcon.entity.v0.feed.ExtractMethod;
 import org.apache.falcon.entity.v0.feed.Feed;
-import org.apache.falcon.entity.v0.feed.FieldsType;
 import org.apache.falcon.entity.v0.feed.FieldIncludeExclude;
+import org.apache.falcon.entity.v0.feed.FieldsType;
 import org.apache.falcon.entity.v0.feed.Import;
 import org.apache.falcon.entity.v0.feed.Lifecycle;
 import org.apache.falcon.entity.v0.feed.Location;
-import org.apache.falcon.entity.v0.feed.Locations;
 import org.apache.falcon.entity.v0.feed.LocationType;
+import org.apache.falcon.entity.v0.feed.Locations;
 import org.apache.falcon.entity.v0.feed.MergeType;
-import org.apache.falcon.entity.v0.feed.Source;
 import org.apache.falcon.entity.v0.feed.RetentionStage;
+import org.apache.falcon.entity.v0.feed.Source;
 import org.apache.falcon.entity.v0.feed.Validity;
-
 import org.apache.falcon.entity.v0.process.Input;
 import org.apache.falcon.entity.v0.process.Inputs;
 import org.apache.falcon.entity.v0.process.Output;
@@ -290,21 +289,33 @@ public class FeedHelperTest extends AbstractTestBase {
 
     @DataProvider(name = "fsPathsforDate")
     public Object[][] createPathsForGetDate() {
+        final TimeZone utc = TimeZone.getTimeZone("UTC");
+        final TimeZone pacificTime = TimeZone.getTimeZone("america/los_angeles");
+        final TimeZone ist = TimeZone.getTimeZone("IST");
+
         return new Object[][] {
-            {"/data/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}", "/data/2015/01/01/00/30",
"2015-01-01T00:30Z"},
-            {"/data/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}", "/data/2015-01-01-01-00",
"2015-01-01T01:00Z"},
-            {"/data/${YEAR}/${MONTH}/${DAY}", "/data/2015/01/01", "2015-01-01T00:00Z"},
-            {"/data/${YEAR}/${MONTH}/${DAY}/data", "/data/2015/01/01/data", "2015-01-01T00:00Z"},
-            {"/data/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}", "/data/2015-01-01/00/30",
null},
-            {"/data/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}/data", "/data/2015-01-01/00/30",
null},
-            {"/data/${YEAR}/${MONTH}/${DAY}/${HOUR}/data", "/data/2015/05/25/00/data/{p1}/p2",
"2015-05-25T00:00Z"},
-            {"/data/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}/data", "/data/2015/05/25/00/00/{p1}/p2",
null},
+            {"/data/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}", "/data/2015/01/01/00/30",
utc, "2015-01-01T00:30Z"},
+            {"/data/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}", "/data/2015-01-01-01-00",
utc, "2015-01-01T01:00Z"},
+            {"/data/${YEAR}/${MONTH}/${DAY}", "/data/2015/01/01", utc, "2015-01-01T00:00Z"},
+            {"/data/${YEAR}/${MONTH}/${DAY}/data", "/data/2015/01/01/data", utc, "2015-01-01T00:00Z"},
+            {"/data/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}", "/data/2015-01-01/00/30",
utc, null},
+            {"/data/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}/data", "/data/2015-01-01/00/30",
utc, null},
+            {"/d/${YEAR}/${MONTH}/${DAY}/${HOUR}/data", "/d/2015/05/25/00/data/{p1}/p2",
utc, "2015-05-25T00:00Z"},
+            {"/data/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}/data", "/data/2015/05/25/00/00/{p1}/p2",
utc, null},
+            {"/d/${YEAR}/${MONTH}/M", "/d/2015/11/M", utc, "2015-11-01T00:00Z"},
+            {"/d/${YEAR}/${MONTH}/${DAY}/M", "/d/2015/11/02/M", utc, "2015-11-02T00:00Z"},
+            {"/d/${YEAR}/${MONTH}/${DAY}/${HOUR}/M", "/d/2015/11/01/04/M", utc, "2015-11-01T04:00Z"},
+            {"/d/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}/M", "/d/2015/11/01/04/15/M", utc,
"2015-11-01T04:15Z"},
+            {"/d/${YEAR}/${MONTH}/M", "/d/2015/11/M", pacificTime, "2015-11-01T07:00Z"},
+            {"/d/${YEAR}/${MONTH}/${DAY}/M", "/d/2015/11/02/M", pacificTime, "2015-11-02T08:00Z"},
+            {"/d/${YEAR}/${MONTH}/${DAY}/${HOUR}/M", "/d/2015/11/01/04/M", pacificTime, "2015-11-01T12:00Z"},
+            {"/d/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}/M", "/d/2015/11/01/04/15/M", ist,
"2015-10-31T22:45Z"},
         };
     }
 
     @Test(dataProvider = "fsPathsforDate")
-    public void testGetDateFromPath(String template, String path, String expectedDate) throws
Exception {
-        Date date = FeedHelper.getDate(template, new Path(path), UTC);
+    public void testGetDateFromPath(String template, String path, TimeZone tz, String expectedDate)
throws Exception {
+        Date date = FeedHelper.getDate(template, new Path(path), tz);
         Assert.assertEquals(SchemaHelper.formatDateUTC(date), expectedDate);
     }
 


Mime
View raw message