falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ajayyad...@apache.org
Subject falcon git commit: FALCON-1560 Lifecycle does not allow feed with frequency greater than days(1). Contributed by Ajay Yadava.
Date Thu, 29 Oct 2015 06:04:16 GMT
Repository: falcon
Updated Branches:
  refs/heads/master 89040a296 -> 44d5028f6


FALCON-1560 Lifecycle does not allow feed with frequency greater than days(1). Contributed
by Ajay Yadava.


Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/44d5028f
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/44d5028f
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/44d5028f

Branch: refs/heads/master
Commit: 44d5028f6a9e0994e6658611fcf9d9b17e5d39a6
Parents: 89040a2
Author: Ajay Yadav <ajay.yadav@inmobi.com>
Authored: Thu Oct 29 11:01:19 2015 +0530
Committer: Ajay Yadav <ajay.yadav@inmobi.com>
Committed: Thu Oct 29 11:01:19 2015 +0530

----------------------------------------------------------------------
 CHANGES.txt                                         |  2 ++
 .../java/org/apache/falcon/entity/FeedHelper.java   | 10 ++++++----
 .../org/apache/falcon/entity/FeedHelperTest.java    | 11 +++++++++--
 .../falcon/entity/parser/FeedEntityParserTest.java  | 16 ++++++++++++++++
 4 files changed, 33 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/falcon/blob/44d5028f/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index b5980be..9305340 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -93,6 +93,8 @@ Branch : 0.8 (Proposed Release version: 0.8)
     FALCON-1403 Revisit IT cleanup and teardown(Narayan Periwal via Pallavi Rao)
 
   BUG FIXES
+    FALCON-1560 Lifecycle does not allow feed with frequency greater than days(1) (Ajay Yadava).
+
     FALCON-1556 Falcon build fails when building with hivedr profile(Sowmya Ramesh)
 
     FALCON-1558 Sla API shows pending instance although _SUCCESS directory is present(Pragya
Mittal via Ajay Yadava).

http://git-wip-us.apache.org/repos/asf/falcon/blob/44d5028f/common/src/main/java/org/apache/falcon/entity/FeedHelper.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/FeedHelper.java b/common/src/main/java/org/apache/falcon/entity/FeedHelper.java
index 2c65eba..0b2e94f 100644
--- a/common/src/main/java/org/apache/falcon/entity/FeedHelper.java
+++ b/common/src/main/java/org/apache/falcon/entity/FeedHelper.java
@@ -50,6 +50,7 @@ import org.apache.falcon.resource.EntityList;
 import org.apache.falcon.resource.FeedInstanceResult;
 import org.apache.falcon.resource.SchedulableEntityInstance;
 import org.apache.falcon.util.BuildProperties;
+import org.apache.falcon.util.DateUtil;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.slf4j.Logger;
@@ -986,11 +987,12 @@ public final class FeedHelper {
         if (retentionStage != null && retentionStage.getFrequency() != null) {
             retentionFrequency = retentionStage.getFrequency();
         } else {
-            Frequency.TimeUnit timeUnit = feed.getFrequency().getTimeUnit();
-            if (timeUnit == Frequency.TimeUnit.hours || timeUnit == Frequency.TimeUnit.minutes)
{
-                retentionFrequency = new Frequency("hours(6)");
+            Frequency feedFrequency = feed.getFrequency();
+            Frequency defaultFrequency = new Frequency("hours(6)");
+            if (DateUtil.getFrequencyInMillis(feedFrequency) < DateUtil.getFrequencyInMillis(defaultFrequency))
{
+                retentionFrequency = defaultFrequency;
             } else {
-                retentionFrequency = new Frequency("days(1)");
+                retentionFrequency = new Frequency(feedFrequency.toString());
             }
         }
         return  retentionFrequency;

http://git-wip-us.apache.org/repos/asf/falcon/blob/44d5028f/common/src/test/java/org/apache/falcon/entity/FeedHelperTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/falcon/entity/FeedHelperTest.java b/common/src/test/java/org/apache/falcon/entity/FeedHelperTest.java
index 60bf1be..10dac49 100644
--- a/common/src/test/java/org/apache/falcon/entity/FeedHelperTest.java
+++ b/common/src/test/java/org/apache/falcon/entity/FeedHelperTest.java
@@ -832,15 +832,22 @@ public class FeedHelperTest extends AbstractTestBase {
         cluster.setName("cluster1");
         clusters.getClusters().add(cluster);
         feed.setClusters(clusters);
-        Assert.assertEquals(FeedHelper.getRetentionFrequency(feed, cluster.getName()), new
Frequency("days(1)"));
+        Assert.assertEquals(FeedHelper.getRetentionFrequency(feed, cluster.getName()), new
Frequency("days(10)"));
 
         feed.setFrequency(new Frequency("hours(1)"));
         Assert.assertEquals(FeedHelper.getRetentionFrequency(feed, cluster.getName()), new
Frequency("hours(6)"));
 
-        feed.setFrequency(new Frequency("minutes(5)"));
+        feed.setFrequency(new Frequency("minutes(10)"));
         Assert.assertEquals(FeedHelper.getRetentionFrequency(feed, cluster.getName()), new
Frequency("hours(6)"));
 
+        feed.setFrequency(new Frequency("hours(7)"));
+        Assert.assertEquals(FeedHelper.getRetentionFrequency(feed, cluster.getName()), new
Frequency("hours(7)"));
+
+        feed.setFrequency(new Frequency("days(2)"));
+        Assert.assertEquals(FeedHelper.getRetentionFrequency(feed, cluster.getName()), new
Frequency("days(2)"));
+
         // lifecycle at both level - retention only at global level.
+        feed.setFrequency(new Frequency("hours(1)"));
         Lifecycle globalLifecycle = new Lifecycle();
         RetentionStage globalRetentionStage = new RetentionStage();
         globalRetentionStage.setFrequency(new Frequency("hours(2)"));

http://git-wip-us.apache.org/repos/asf/falcon/blob/44d5028f/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java
b/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java
index 7f4abce..a583f52 100644
--- a/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java
+++ b/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java
@@ -208,6 +208,22 @@ public class FeedEntityParserTest extends AbstractTestBase {
         parser.validate(feed); // no validation exception should be thrown
     }
 
+    @Test
+    public void testDefaultRetentionFrequencyConflict() throws Exception {
+        Feed feed = parser.parseAndValidate(this.getClass().getResourceAsStream(FEED3_XML));
+        feed.getLifecycle().getRetentionStage().setFrequency(null);
+        feed.getClusters().getClusters().get(0).getLifecycle().getRetentionStage().setFrequency(null);
+        feed.setFrequency(Frequency.fromString("minutes(10)"));
+        parser.validate(feed); // shouldn't throw a validation exception
+
+
+        feed.setFrequency(Frequency.fromString("hours(7)"));
+        parser.validate(feed); // shouldn't throw a validation exception
+
+        feed.setFrequency(Frequency.fromString("days(2)"));
+        parser.validate(feed); // shouldn't throw a validation exception
+    }
+
     @Test(expectedExceptions = ValidationException.class,
         expectedExceptionsMessageRegExp = ".*Retention can not be more frequent than data
availability.*")
     public void testRetentionFrequentThanFeed() throws Exception {


Mime
View raw message