Return-Path: X-Original-To: apmail-falcon-commits-archive@minotaur.apache.org Delivered-To: apmail-falcon-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 2337E18575 for ; Thu, 14 Jan 2016 10:53:01 +0000 (UTC) Received: (qmail 97930 invoked by uid 500); 14 Jan 2016 10:53:01 -0000 Delivered-To: apmail-falcon-commits-archive@falcon.apache.org Received: (qmail 97860 invoked by uid 500); 14 Jan 2016 10:53:01 -0000 Mailing-List: contact commits-help@falcon.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@falcon.apache.org Delivered-To: mailing list commits@falcon.apache.org Received: (qmail 97834 invoked by uid 99); 14 Jan 2016 10:53:00 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 14 Jan 2016 10:53:00 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 8D895E3861; Thu, 14 Jan 2016 10:53:00 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: pallavi@apache.org To: commits@falcon.apache.org Date: Thu, 14 Jan 2016 10:53:01 -0000 Message-Id: In-Reply-To: <1e21129fa1b6492fae226190c86fc8da@git.apache.org> References: <1e21129fa1b6492fae226190c86fc8da@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [2/2] falcon git commit: FALCON-1675 Lifecycle stage is not honoured when only cluster level lifecycle is implemented (by Ajay Yadava) FALCON-1675 Lifecycle stage is not honoured when only cluster level lifecycle is implemented (by Ajay Yadava) Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/583651c0 Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/583651c0 Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/583651c0 Branch: refs/heads/master Commit: 583651c00d1425b1156b2da92fe8f95148821a84 Parents: 8dfc2c9 Author: Pallavi Rao Authored: Thu Jan 14 15:28:32 2016 +0530 Committer: Pallavi Rao Committed: Thu Jan 14 15:28:32 2016 +0530 ---------------------------------------------------------------------- CHANGES.txt | 2 + .../falcon/oozie/feed/FeedBundleBuilder.java | 13 ++--- .../feed/OozieFeedWorkflowBuilderTest.java | 29 +++++++++- .../feed/fs-local-retention-lifecycle-feed.xml | 61 ++++++++++++++++++++ 4 files changed, 95 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/583651c0/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index cf97ca2..590f17a 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -112,6 +112,8 @@ Proposed Release Version: 0.9 OPTIMIZATIONS BUG FIXES + FALCON-1675 Lifecycle stage is not honoured when only cluster level lifecycle is implemented (Ajay Yadava via Pallavi Rao) + FALCON-1730 Upgrade enunciate for JDK 8 compatibility(Deepak Barr via Ajay Yadava) FALCON-1741 Rerun API behaviour different in case of succeeded instances(Pallavi Rao via Ajay Yadava) http://git-wip-us.apache.org/repos/asf/falcon/blob/583651c0/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedBundleBuilder.java ---------------------------------------------------------------------- diff --git a/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedBundleBuilder.java b/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedBundleBuilder.java index 1205d91..c758411 100644 --- a/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedBundleBuilder.java +++ b/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedBundleBuilder.java @@ -18,22 +18,21 @@ package org.apache.falcon.oozie.feed; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + import org.apache.falcon.FalconException; import org.apache.falcon.Tag; import org.apache.falcon.entity.FeedHelper; import org.apache.falcon.entity.v0.cluster.Cluster; import org.apache.falcon.entity.v0.feed.Feed; -import org.apache.falcon.entity.v0.feed.Lifecycle; import org.apache.falcon.lifecycle.LifecyclePolicy; import org.apache.falcon.oozie.OozieBundleBuilder; import org.apache.falcon.oozie.OozieCoordinatorBuilder; import org.apache.falcon.service.LifecyclePolicyMap; import org.apache.hadoop.fs.Path; -import java.util.ArrayList; -import java.util.List; -import java.util.Properties; - /** * Builds oozie bundle for the feed. */ @@ -46,8 +45,8 @@ public class FeedBundleBuilder extends OozieBundleBuilder { protected List buildCoords(Cluster cluster, Path buildPath) throws FalconException { // if feed has lifecycle defined - then use it to create coordinator and wf else fall back List props = new ArrayList<>(); - Lifecycle lifecycle = this.entity.getLifecycle(); - if (lifecycle != null) { + boolean isLifeCycleEnabled = FeedHelper.isLifecycleEnabled(this.entity, cluster.getName()); + if (isLifeCycleEnabled) { for (String name : FeedHelper.getPolicies(this.entity, cluster.getName())) { LifecyclePolicy policy = LifecyclePolicyMap.get().get(name); if (policy == null) { http://git-wip-us.apache.org/repos/asf/falcon/blob/583651c0/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java ---------------------------------------------------------------------- diff --git a/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java b/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java index 9388c68..a553ea7 100644 --- a/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java +++ b/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java @@ -93,7 +93,7 @@ public class OozieFeedWorkflowBuilderTest extends AbstractTestBase { private Feed tableFeed; private Feed fsReplFeed; private Feed lifecycleRetentionFeed; - private Feed retentionFeed; + private Feed lifecycleLocalRetentionFeed; private Feed fsReplFeedCounter; private static final String SRC_CLUSTER_PATH = "/feed/src-cluster.xml"; @@ -102,7 +102,7 @@ public class OozieFeedWorkflowBuilderTest extends AbstractTestBase { private static final String TABLE_FEED = "/feed/table-replication-feed.xml"; private static final String FS_REPLICATION_FEED = "/feed/fs-replication-feed.xml"; private static final String FS_RETENTION_LIFECYCLE_FEED = "/feed/fs-retention-lifecycle-feed.xml"; - private static final String FS_RETENTION_ORIG_FEED = "/feed/fs-retention-feed.xml"; + private static final String FS_LOCAL_RETENTION_LIFECYCLE_FEED = "/feed/fs-local-retention-lifecycle-feed.xml"; private static final String FS_REPLICATION_FEED_COUNTER = "/feed/fs-replication-feed-counters.xml"; @BeforeClass @@ -137,7 +137,7 @@ public class OozieFeedWorkflowBuilderTest extends AbstractTestBase { fsReplFeedCounter = (Feed) storeEntity(EntityType.FEED, FS_REPLICATION_FEED_COUNTER); tableFeed = (Feed) storeEntity(EntityType.FEED, TABLE_FEED); lifecycleRetentionFeed = (Feed) storeEntity(EntityType.FEED, FS_RETENTION_LIFECYCLE_FEED); - retentionFeed = (Feed) storeEntity(EntityType.FEED, FS_RETENTION_ORIG_FEED); + lifecycleLocalRetentionFeed = (Feed) storeEntity(EntityType.FEED, FS_LOCAL_RETENTION_LIFECYCLE_FEED); } private Entity storeEntity(EntityType type, String resource) throws Exception { @@ -198,6 +198,29 @@ public class OozieFeedWorkflowBuilderTest extends AbstractTestBase { Assert.assertEquals(wfProps.get("jobPriority"), "LOW"); } + @Test + public void testLocalOnlyRetentionLifecycle() throws Exception { + OozieEntityBuilder builder = OozieEntityBuilder.get(lifecycleLocalRetentionFeed); + Path bundlePath = new Path("/projects/falcon/"); + builder.build(trgCluster, bundlePath); + + BUNDLEAPP bundle = getBundle(trgMiniDFS.getFileSystem(), bundlePath); + List coords = bundle.getCoordinator(); + COORDINATORAPP coord = getCoordinator(trgMiniDFS, coords.get(0).getAppPath()); + assertLibExtensions(coord, "retention"); + HashMap props = getCoordProperties(coord); + Assert.assertEquals(props.get("ENTITY_PATH"), bundlePath.toString() + "/RETENTION"); + Assert.assertEquals(coord.getFrequency(), "${coord:hours(12)}"); + Assert.assertEquals(coord.getTimezone(), "UTC"); + + HashMap wfProps = getWorkflowProperties(trgMiniDFS.getFileSystem(), coord); + Assert.assertEquals(wfProps.get("feedNames"), lifecycleLocalRetentionFeed.getName()); + Assert.assertTrue(StringUtils.equals(wfProps.get("entityType"), EntityType.FEED.name())); + Assert.assertEquals(wfProps.get("userWorkflowEngine"), "falcon"); + Assert.assertEquals(wfProps.get("queueName"), "local"); + Assert.assertEquals(wfProps.get("limit"), "hours(4)"); + Assert.assertEquals(wfProps.get("jobPriority"), "HIGH"); + } @Test public void testRetentionFrequency() throws Exception { http://git-wip-us.apache.org/repos/asf/falcon/blob/583651c0/oozie/src/test/resources/feed/fs-local-retention-lifecycle-feed.xml ---------------------------------------------------------------------- diff --git a/oozie/src/test/resources/feed/fs-local-retention-lifecycle-feed.xml b/oozie/src/test/resources/feed/fs-local-retention-lifecycle-feed.xml new file mode 100644 index 0000000..bdf1e59 --- /dev/null +++ b/oozie/src/test/resources/feed/fs-local-retention-lifecycle-feed.xml @@ -0,0 +1,61 @@ + + + + + + + + + + minutes(5) + UTC + + + + + + + + + hours(12) + local + HIGH + + + + + + + + + + + + + + + + + + + + + + +