falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From pall...@apache.org
Subject falcon git commit: FALCON-2116 Queue defined in <retention-stage> is not getting used in actual retention job
Date Wed, 17 Aug 2016 04:00:20 GMT
Repository: falcon
Updated Branches:
  refs/heads/master 7354f870b -> 3a7c993b5


FALCON-2116 Queue defined in <retention-stage> is not getting used in actual retention
job

Author: sandeep <sandysmdl@gmail.com>

Reviewers: @pallavi-rao

Closes #266 from sandeepSamudrala/FALCON-2116 and squashes the following commits:

a23fe90 [sandeep] FALCON-2116 Queue defined in <retention-stage> is not getting used
in actual retention job
d6dc8bf [sandeep] Merge branch 'master' of https://github.com/apache/falcon
1bb8d3c [sandeep] Merge branch 'master' of https://github.com/apache/falcon
c065566 [sandeep] reverting last line changes made
1a4dcd2 [sandeep] rebased and resolved the conflicts from master
271318b [sandeep] FALCON-2097. Adding UT to the new method for getting next instance time
with Delay.
a94d4fe [sandeep] rebasing from master
9e68a57 [sandeep] FALCON-298. Feed update with replication delay creates holes


Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/3a7c993b
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/3a7c993b
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/3a7c993b

Branch: refs/heads/master
Commit: 3a7c993b55564d11ae79d25d32fd5fbd63ce4fe9
Parents: 7354f87
Author: sandeep <sandysmdl@gmail.com>
Authored: Wed Aug 17 09:30:13 2016 +0530
Committer: Pallavi Rao <pallavi.rao@inmobi.com>
Committed: Wed Aug 17 09:30:13 2016 +0530

----------------------------------------------------------------------
 .../engine/oozie/retention/AgeBasedCoordinatorBuilder.java   | 8 ++++++++
 .../falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java      | 3 ++-
 .../src/test/resources/feed/fs-retention-lifecycle-feed.xml  | 3 ++-
 3 files changed, 12 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/falcon/blob/3a7c993b/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/retention/AgeBasedCoordinatorBuilder.java
----------------------------------------------------------------------
diff --git a/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/retention/AgeBasedCoordinatorBuilder.java
b/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/retention/AgeBasedCoordinatorBuilder.java
index 7a25d86..f7b31e0 100644
--- a/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/retention/AgeBasedCoordinatorBuilder.java
+++ b/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/retention/AgeBasedCoordinatorBuilder.java
@@ -41,6 +41,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.util.Date;
+import java.util.List;
 import java.util.Properties;
 
 /**
@@ -100,6 +101,13 @@ public final class AgeBasedCoordinatorBuilder {
         Path buildPath = OozieBuilderUtils.getBuildPath(basePath, LifeCycle.EVICTION.getTag());
         Properties props = OozieBuilderUtils.createCoordDefaultConfiguration(coordName, feed);
         props.putAll(OozieBuilderUtils.getProperties(buildPath, coordName));
+        props.putAll(EntityUtil.getEntityProperties(feed));
+        props.put("queueName", FeedHelper.getLifecycleRetentionQueue(feed, cluster.getName()));
+        List<org.apache.falcon.entity.v0.feed.Property> retentionProperties =
+                FeedHelper.getLifecycle(feed, cluster.getName()).getRetentionStage().getProperties().getProperties();
+        for (org.apache.falcon.entity.v0.feed.Property retentionProperty : retentionProperties)
{
+            props.put(retentionProperty.getName(), retentionProperty.getValue());
+        }
 
         WORKFLOW workflow = new WORKFLOW();
         String entityPath = wfProp.getProperty(OozieBuilderUtils.ENTITY_PATH);

http://git-wip-us.apache.org/repos/asf/falcon/blob/3a7c993b/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java
----------------------------------------------------------------------
diff --git a/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java
b/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java
index fde5532..08d3339 100644
--- a/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java
+++ b/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java
@@ -189,6 +189,7 @@ public class OozieFeedWorkflowBuilderTest extends AbstractTestBase {
         assertLibExtensions(coord, "retention");
         HashMap<String, String> props = getCoordProperties(coord);
         Assert.assertEquals(props.get("ENTITY_PATH"), bundlePath.toString() + "/RETENTION");
+        Assert.assertEquals(props.get("queueName"), "ageBasedDeleteQueue");
         Assert.assertEquals(coord.getFrequency(), "${coord:hours(17)}");
         Assert.assertEquals(coord.getEnd(), endTime);
         Assert.assertEquals(coord.getTimezone(), "UTC");
@@ -197,7 +198,7 @@ public class OozieFeedWorkflowBuilderTest extends AbstractTestBase {
         Assert.assertEquals(wfProps.get("feedNames"), lifecycleRetentionFeed.getName());
         Assert.assertTrue(StringUtils.equals(wfProps.get("entityType"), EntityType.FEED.name()));
         Assert.assertEquals(wfProps.get("userWorkflowEngine"), "falcon");
-        Assert.assertEquals(wfProps.get("queueName"), "retention");
+        Assert.assertEquals(wfProps.get("queueName"), "ageBasedDeleteQueue");
         Assert.assertEquals(wfProps.get("limit"), "hours(2)");
         Assert.assertEquals(wfProps.get("jobPriority"), "LOW");
     }

http://git-wip-us.apache.org/repos/asf/falcon/blob/3a7c993b/oozie/src/test/resources/feed/fs-retention-lifecycle-feed.xml
----------------------------------------------------------------------
diff --git a/oozie/src/test/resources/feed/fs-retention-lifecycle-feed.xml b/oozie/src/test/resources/feed/fs-retention-lifecycle-feed.xml
index 2cadfe0..a230f15 100644
--- a/oozie/src/test/resources/feed/fs-retention-lifecycle-feed.xml
+++ b/oozie/src/test/resources/feed/fs-retention-lifecycle-feed.xml
@@ -45,12 +45,13 @@
     <properties>
         <property name="maxMaps" value="33" />
         <property name="mapBandwidth" value="2" />
+        <property name="queueName" value="retentionQueue" />
     </properties>
 
     <lifecycle>
         <retention-stage>
             <frequency>hours(17)</frequency>
-            <queue>retention</queue>
+            <queue>ageBasedDeleteQueue</queue>
             <priority>LOW</priority>
             <properties>
                 <property name="retention.policy.agebaseddelete.limit" value="hours(2)"></property>


Mime
View raw message