falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sowmya...@apache.org
Subject falcon git commit: FALCON-1569 Bug in setting the frequency of Feed retention coordinator. Contributed by Sowmya Ramesh
Date Mon, 09 Nov 2015 21:43:51 GMT
Repository: falcon
Updated Branches:
  refs/heads/master aa18bfdae -> d97052f34


FALCON-1569 Bug in setting the frequency of Feed retention coordinator. Contributed by Sowmya
Ramesh


Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/d97052f3
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/d97052f3
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/d97052f3

Branch: refs/heads/master
Commit: d97052f3423fae8213e4e3bf677ee21b632b82b0
Parents: aa18bfd
Author: Sowmya Ramesh <sramesh@hortonworks.com>
Authored: Mon Nov 9 13:43:44 2015 -0800
Committer: Sowmya Ramesh <sramesh@hortonworks.com>
Committed: Mon Nov 9 13:43:44 2015 -0800

----------------------------------------------------------------------
 CHANGES.txt                                     |  2 +
 .../feed/FeedRetentionCoordinatorBuilder.java   |  8 ++--
 .../feed/OozieFeedWorkflowBuilderTest.java      | 39 ++++++++++++++++++++
 .../apache/falcon/unit/FalconUnitTestBase.java  |  6 ++-
 4 files changed, 50 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/falcon/blob/d97052f3/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 586847e..5a964da 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -27,6 +27,8 @@ Trunk (Unreleased)
   OPTIMIZATIONS
 
   BUG FIXES
+    FALCON-1569 Bug in setting the frequency of Feed retention coordinator(Sowmya Ramesh)
+
     FALCON-1578 post-processing action fails with javax.servlet.jsp.el.ELException(Venkat
Ramachandran via Ajay Yadava)
 
     FALCON-1579 post-processing action fails with javax.servlet.jsp.el.ELException (Sowmya
Ramesh via Pallavi Rao)

http://git-wip-us.apache.org/repos/asf/falcon/blob/d97052f3/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedRetentionCoordinatorBuilder.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedRetentionCoordinatorBuilder.java
b/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedRetentionCoordinatorBuilder.java
index 0809bb6..69ca2c3 100644
--- a/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedRetentionCoordinatorBuilder.java
+++ b/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedRetentionCoordinatorBuilder.java
@@ -24,7 +24,7 @@ import org.apache.falcon.LifeCycle;
 import org.apache.falcon.Tag;
 import org.apache.falcon.entity.EntityUtil;
 import org.apache.falcon.entity.FeedHelper;
-import org.apache.falcon.entity.v0.Frequency.TimeUnit;
+import org.apache.falcon.entity.v0.Frequency;
 import org.apache.falcon.entity.v0.SchemaHelper;
 import org.apache.falcon.entity.v0.cluster.Cluster;
 import org.apache.falcon.entity.v0.feed.Feed;
@@ -34,6 +34,7 @@ import org.apache.falcon.oozie.OozieOrchestrationWorkflowBuilder;
 import org.apache.falcon.oozie.coordinator.ACTION;
 import org.apache.falcon.oozie.coordinator.COORDINATORAPP;
 import org.apache.falcon.oozie.coordinator.WORKFLOW;
+import org.apache.falcon.util.DateUtil;
 import org.apache.hadoop.fs.Path;
 
 import java.util.Arrays;
@@ -67,8 +68,9 @@ public class FeedRetentionCoordinatorBuilder extends OozieCoordinatorBuilder<Fee
             coord.setStart(SchemaHelper.formatDateUTC(new Date()));
         }
         coord.setTimezone(entity.getTimezone().getID());
-        TimeUnit timeUnit = entity.getFrequency().getTimeUnit();
-        if (timeUnit == TimeUnit.hours || timeUnit == TimeUnit.minutes) {
+        Frequency entityFrequency = entity.getFrequency();
+        Frequency defaultFrequency = new Frequency("hours(24)");
+        if (DateUtil.getFrequencyInMillis(entityFrequency) < DateUtil.getFrequencyInMillis(defaultFrequency))
{
             coord.setFrequency("${coord:hours(6)}");
         } else {
             coord.setFrequency("${coord:days(1)}");

http://git-wip-us.apache.org/repos/asf/falcon/blob/d97052f3/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java
----------------------------------------------------------------------
diff --git a/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java
b/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java
index 5e93027..176a15e 100644
--- a/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java
+++ b/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java
@@ -29,6 +29,7 @@ import org.apache.falcon.entity.Storage;
 import org.apache.falcon.entity.store.ConfigurationStore;
 import org.apache.falcon.entity.v0.Entity;
 import org.apache.falcon.entity.v0.EntityType;
+import org.apache.falcon.entity.v0.Frequency;
 import org.apache.falcon.entity.v0.SchemaHelper;
 import org.apache.falcon.entity.v0.cluster.Cluster;
 import org.apache.falcon.entity.v0.cluster.ClusterLocationType;
@@ -187,6 +188,44 @@ public class OozieFeedWorkflowBuilderTest extends AbstractTestBase {
     }
 
     @Test
+    public void testRetentionFrequency() throws Exception {
+        feed.setFrequency(new Frequency("minutes(36000)"));
+        buildCoordAndValidateFrequency("${coord:days(1)}");
+
+        feed.setFrequency(new Frequency("hours(2)"));
+        buildCoordAndValidateFrequency("${coord:hours(6)}");
+
+        feed.setFrequency(new Frequency("minutes(50)"));
+        buildCoordAndValidateFrequency("${coord:hours(6)}");
+
+        feed.setFrequency(new Frequency("days(1)"));
+        buildCoordAndValidateFrequency("${coord:days(1)}");
+    }
+
+    private void buildCoordAndValidateFrequency(final String expectedFrequency) throws Exception
{
+        // retention on src cluster
+        OozieCoordinatorBuilder builder = OozieCoordinatorBuilder.get(feed, Tag.RETENTION);
+        List<Properties> srcCoords = builder.buildCoords(
+                srcCluster, new Path("/projects/falcon/"));
+        COORDINATORAPP srcCoord = getCoordinator(srcMiniDFS, srcCoords.get(0).getProperty(OozieEntityBuilder
+                .ENTITY_PATH));
+
+        // Assert src coord frequency
+        Assert.assertEquals(srcCoord.getFrequency(), expectedFrequency);
+
+        // retention on target cluster
+        OozieEntityBuilder entityBuilder = OozieEntityBuilder.get(feed);
+        Path bundlePath = new Path("/projects/falcon/");
+        entityBuilder.build(trgCluster, bundlePath);
+        BUNDLEAPP bundle = getBundle(trgMiniDFS.getFileSystem(), bundlePath);
+        List<COORDINATOR> coords = bundle.getCoordinator();
+
+        COORDINATORAPP tgtCoord = getCoordinator(trgMiniDFS, coords.get(0).getAppPath());
+        // Assert target coord frequency
+        Assert.assertEquals(tgtCoord.getFrequency(), expectedFrequency);
+    }
+
+    @Test
     public void testReplicationCoordsForFSStorage() throws Exception {
         OozieEntityBuilder builder = OozieEntityBuilder.get(feed);
         Path bundlePath = new Path("/projects/falcon/");

http://git-wip-us.apache.org/repos/asf/falcon/blob/d97052f3/unit/src/test/java/org/apache/falcon/unit/FalconUnitTestBase.java
----------------------------------------------------------------------
diff --git a/unit/src/test/java/org/apache/falcon/unit/FalconUnitTestBase.java b/unit/src/test/java/org/apache/falcon/unit/FalconUnitTestBase.java
index bd81798..af5a3c5 100644
--- a/unit/src/test/java/org/apache/falcon/unit/FalconUnitTestBase.java
+++ b/unit/src/test/java/org/apache/falcon/unit/FalconUnitTestBase.java
@@ -350,11 +350,13 @@ public class FalconUnitTestBase {
     public InstancesResult.WorkflowStatus getRetentionStatus(String feedName, String cluster)
throws FalconException,
             FalconCLIException {
         Feed feedEntity = EntityUtil.getEntity(EntityType.FEED, feedName);
-        Frequency.TimeUnit timeUnit = feedEntity.getFrequency().getTimeUnit();
+
+        Frequency feedFrequency = feedEntity.getFrequency();
+        Frequency defaultFrequency = new Frequency("hours(24)");
         long endTimeInMillis = System.currentTimeMillis() + 30000;
         String endTime = DateUtil.getDateFormatFromTime(endTimeInMillis);
         long startTimeInMillis;
-        if (timeUnit == Frequency.TimeUnit.hours || timeUnit == Frequency.TimeUnit.minutes)
{
+        if (DateUtil.getFrequencyInMillis(feedFrequency) < DateUtil.getFrequencyInMillis(defaultFrequency))
{
             startTimeInMillis = endTimeInMillis - (6 * DateUtil.HOUR_IN_MILLIS);
         } else {
             startTimeInMillis = endTimeInMillis - (24 * DateUtil.HOUR_IN_MILLIS);


Mime
View raw message