falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From suh...@apache.org
Subject git commit: FALCON-338 late data recording is enabled by default for all feeds irrespective of late arrival config. Contributed by Ajay Yadav
Date Thu, 04 Sep 2014 18:57:49 GMT
Repository: incubator-falcon
Updated Branches:
  refs/heads/master d9c115e0e -> 6f770a34d


FALCON-338 late data recording is enabled by default for all feeds irrespective of late arrival
config. Contributed by Ajay Yadav


Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/6f770a34
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/6f770a34
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/6f770a34

Branch: refs/heads/master
Commit: 6f770a34df400f3399300ea371a46c1ae0878b06
Parents: d9c115e
Author: Suhas V <suhas.v@inmobi.com>
Authored: Fri Sep 5 00:24:18 2014 +0530
Committer: Suhas V <suhas.v@inmobi.com>
Committed: Fri Sep 5 00:24:18 2014 +0530

----------------------------------------------------------------------
 CHANGES.txt                                           |  3 +++
 .../java/org/apache/falcon/entity/EntityUtil.java     |  5 +++++
 .../java/org/apache/falcon/entity/EntityUtilTest.java | 14 ++++++++++++++
 .../falcon/rerun/handler/TestLateRerunHandler.java    | 13 +++++++++++++
 4 files changed, 35 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/6f770a34/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 2086f5b..3e9dcf6 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -77,6 +77,9 @@ Trunk (Unreleased)
   OPTIMIZATIONS
 
   BUG FIXES
+   FALCON-338 - late data recording is enabled by default for all feeds 
+   irrespective of late arrival config (Ajay Yadav via Suhas Vasu)
+
    FALCON-652 EntityUtils tests are failing (Ajay Yadav via Venkatesh Seetharam)
 
    FALCON-650 Instance list APIs occassionally fail when orderBy set to

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/6f770a34/common/src/main/java/org/apache/falcon/entity/EntityUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/EntityUtil.java b/common/src/main/java/org/apache/falcon/entity/EntityUtil.java
index 8f258fb..9b66aed 100644
--- a/common/src/main/java/org/apache/falcon/entity/EntityUtil.java
+++ b/common/src/main/java/org/apache/falcon/entity/EntityUtil.java
@@ -613,6 +613,11 @@ public final class EntityUtil {
                 return null;
             }
 
+            //If late Arrival is not configured do not process further
+            if(((Feed) entity).getLateArrival() == null){
+                return null;
+            }
+
             LateProcess lateProcess = new LateProcess();
             lateProcess.setDelay(new Frequency(RuntimeProperties.get().getProperty("feed.late.frequency",
"hours(3)")));
             lateProcess.setPolicy(

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/6f770a34/common/src/test/java/org/apache/falcon/entity/EntityUtilTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/falcon/entity/EntityUtilTest.java b/common/src/test/java/org/apache/falcon/entity/EntityUtilTest.java
index 50ee649..3d501a7 100644
--- a/common/src/test/java/org/apache/falcon/entity/EntityUtilTest.java
+++ b/common/src/test/java/org/apache/falcon/entity/EntityUtilTest.java
@@ -19,10 +19,12 @@
 package org.apache.falcon.entity;
 
 import org.apache.falcon.Pair;
+import org.apache.falcon.FalconException;
 import org.apache.falcon.entity.v0.EntityType;
 import org.apache.falcon.entity.v0.Frequency;
 import org.apache.falcon.entity.v0.SchemaHelper;
 import org.apache.falcon.entity.v0.feed.Feed;
+import org.apache.falcon.entity.v0.feed.LateArrival;
 import org.apache.falcon.entity.v0.process.Cluster;
 import org.apache.falcon.entity.v0.process.Process;
 import org.testng.Assert;
@@ -231,4 +233,16 @@ public class EntityUtilTest extends AbstractTestBase {
         Assert.assertEquals(startEndDates.first, expectedStartDate);
         Assert.assertEquals(startEndDates.second, expectedEndDate);
     }
+
+    @Test
+    public void testGetLateProcessFeed() throws FalconException{
+        Feed feed = new Feed();
+
+        Assert.assertNull(EntityUtil.getLateProcess(feed));
+        LateArrival lateArrival = new LateArrival();
+        lateArrival.setCutOff(Frequency.fromString("days(1)"));
+        feed.setLateArrival(lateArrival);
+        Assert.assertNotNull(EntityUtil.getLateProcess(feed));
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/6f770a34/rerun/src/test/java/org/apache/falcon/rerun/handler/TestLateRerunHandler.java
----------------------------------------------------------------------
diff --git a/rerun/src/test/java/org/apache/falcon/rerun/handler/TestLateRerunHandler.java
b/rerun/src/test/java/org/apache/falcon/rerun/handler/TestLateRerunHandler.java
index 8137f60..93fac13 100644
--- a/rerun/src/test/java/org/apache/falcon/rerun/handler/TestLateRerunHandler.java
+++ b/rerun/src/test/java/org/apache/falcon/rerun/handler/TestLateRerunHandler.java
@@ -47,4 +47,17 @@ public class TestLateRerunHandler {
         cutOff = LateRerunHandler.getCutOffTime(feed, nm);
         Assert.assertTrue(cutOff.after(new Date()));
     }
+
+    /**
+     * This test checks that LateData Handler is invoked only if LateArrival
+     * is configured for a feed.
+     */
+    @Test
+    public void testFeedLateArrivalCheck() throws FalconException {
+        Feed feed = new Feed();
+        String nominalTime = "2013-10-01T12:00Z";
+        Date cutOff = LateRerunHandler.getCutOffTime(feed, nominalTime);
+        Assert.assertEquals(cutOff, new Date(0));
+
+    }
 }


Mime
View raw message