falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From pall...@apache.org
Subject [2/2] falcon git commit: FALCON-1675 Lifecycle stage is not honoured when only cluster level lifecycle is implemented (by Ajay Yadava)
Date Thu, 14 Jan 2016 11:04:41 GMT
FALCON-1675 Lifecycle stage is not honoured when only cluster level lifecycle is implemented
(by Ajay Yadava)


Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/1cfeea39
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/1cfeea39
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/1cfeea39

Branch: refs/heads/0.9
Commit: 1cfeea3909cf6212811f11fe1b523ddf959aad97
Parents: 88354a1
Author: Pallavi Rao <pallavi.rao@inmobi.com>
Authored: Thu Jan 14 16:22:35 2016 +0530
Committer: Pallavi Rao <pallavi.rao@inmobi.com>
Committed: Thu Jan 14 16:22:35 2016 +0530

----------------------------------------------------------------------
 CHANGES.txt                                     |  2 +
 .../falcon/oozie/feed/FeedBundleBuilder.java    | 13 ++---
 .../feed/OozieFeedWorkflowBuilderTest.java      | 29 +++++++++-
 .../feed/fs-local-retention-lifecycle-feed.xml  | 61 ++++++++++++++++++++
 4 files changed, 95 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/falcon/blob/1cfeea39/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index cb0b671..f1c403d 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -111,6 +111,8 @@ Proposed Release Version: 0.9
   OPTIMIZATIONS
 
   BUG FIXES
+    FALCON-1675 Lifecycle stage is not honoured when only cluster level lifecycle is implemented
(Ajay Yadava via Pallavi Rao)
+
     FALCON-1730 Upgrade enunciate for JDK 8 compatibility(Deepak Barr via Ajay Yadava)
 
     FALCON-1741 Rerun API behaviour different in case of succeeded instances(Pallavi Rao
via Ajay Yadava)

http://git-wip-us.apache.org/repos/asf/falcon/blob/1cfeea39/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedBundleBuilder.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedBundleBuilder.java b/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedBundleBuilder.java
index 1205d91..c758411 100644
--- a/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedBundleBuilder.java
+++ b/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedBundleBuilder.java
@@ -18,22 +18,21 @@
 
 package org.apache.falcon.oozie.feed;
 
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+
 import org.apache.falcon.FalconException;
 import org.apache.falcon.Tag;
 import org.apache.falcon.entity.FeedHelper;
 import org.apache.falcon.entity.v0.cluster.Cluster;
 import org.apache.falcon.entity.v0.feed.Feed;
-import org.apache.falcon.entity.v0.feed.Lifecycle;
 import org.apache.falcon.lifecycle.LifecyclePolicy;
 import org.apache.falcon.oozie.OozieBundleBuilder;
 import org.apache.falcon.oozie.OozieCoordinatorBuilder;
 import org.apache.falcon.service.LifecyclePolicyMap;
 import org.apache.hadoop.fs.Path;
 
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Properties;
-
 /**
  * Builds oozie bundle for the feed.
  */
@@ -46,8 +45,8 @@ public class FeedBundleBuilder extends OozieBundleBuilder<Feed> {
     protected List<Properties> buildCoords(Cluster cluster, Path buildPath) throws
FalconException {
         // if feed has lifecycle defined - then use it to create coordinator and wf else
fall back
         List<Properties> props = new ArrayList<>();
-        Lifecycle lifecycle = this.entity.getLifecycle();
-        if (lifecycle != null) {
+        boolean isLifeCycleEnabled = FeedHelper.isLifecycleEnabled(this.entity, cluster.getName());
+        if (isLifeCycleEnabled) {
             for (String name : FeedHelper.getPolicies(this.entity, cluster.getName())) {
                 LifecyclePolicy policy = LifecyclePolicyMap.get().get(name);
                 if (policy == null) {

http://git-wip-us.apache.org/repos/asf/falcon/blob/1cfeea39/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java
----------------------------------------------------------------------
diff --git a/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java
b/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java
index 9388c68..a553ea7 100644
--- a/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java
+++ b/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java
@@ -93,7 +93,7 @@ public class OozieFeedWorkflowBuilderTest extends AbstractTestBase {
     private Feed tableFeed;
     private Feed fsReplFeed;
     private Feed lifecycleRetentionFeed;
-    private Feed retentionFeed;
+    private Feed lifecycleLocalRetentionFeed;
     private Feed fsReplFeedCounter;
 
     private static final String SRC_CLUSTER_PATH = "/feed/src-cluster.xml";
@@ -102,7 +102,7 @@ public class OozieFeedWorkflowBuilderTest extends AbstractTestBase {
     private static final String TABLE_FEED = "/feed/table-replication-feed.xml";
     private static final String FS_REPLICATION_FEED = "/feed/fs-replication-feed.xml";
     private static final String FS_RETENTION_LIFECYCLE_FEED = "/feed/fs-retention-lifecycle-feed.xml";
-    private static final String FS_RETENTION_ORIG_FEED = "/feed/fs-retention-feed.xml";
+    private static final String FS_LOCAL_RETENTION_LIFECYCLE_FEED = "/feed/fs-local-retention-lifecycle-feed.xml";
     private static final String FS_REPLICATION_FEED_COUNTER = "/feed/fs-replication-feed-counters.xml";
 
     @BeforeClass
@@ -137,7 +137,7 @@ public class OozieFeedWorkflowBuilderTest extends AbstractTestBase {
         fsReplFeedCounter = (Feed) storeEntity(EntityType.FEED, FS_REPLICATION_FEED_COUNTER);
         tableFeed = (Feed) storeEntity(EntityType.FEED, TABLE_FEED);
         lifecycleRetentionFeed = (Feed) storeEntity(EntityType.FEED, FS_RETENTION_LIFECYCLE_FEED);
-        retentionFeed = (Feed) storeEntity(EntityType.FEED, FS_RETENTION_ORIG_FEED);
+        lifecycleLocalRetentionFeed = (Feed) storeEntity(EntityType.FEED, FS_LOCAL_RETENTION_LIFECYCLE_FEED);
     }
 
     private Entity storeEntity(EntityType type, String resource) throws Exception {
@@ -198,6 +198,29 @@ public class OozieFeedWorkflowBuilderTest extends AbstractTestBase {
         Assert.assertEquals(wfProps.get("jobPriority"), "LOW");
     }
 
+    @Test
+    public void testLocalOnlyRetentionLifecycle() throws Exception {
+        OozieEntityBuilder builder = OozieEntityBuilder.get(lifecycleLocalRetentionFeed);
+        Path bundlePath = new Path("/projects/falcon/");
+        builder.build(trgCluster, bundlePath);
+
+        BUNDLEAPP bundle = getBundle(trgMiniDFS.getFileSystem(), bundlePath);
+        List<COORDINATOR> coords = bundle.getCoordinator();
+        COORDINATORAPP coord = getCoordinator(trgMiniDFS, coords.get(0).getAppPath());
+        assertLibExtensions(coord, "retention");
+        HashMap<String, String> props = getCoordProperties(coord);
+        Assert.assertEquals(props.get("ENTITY_PATH"), bundlePath.toString() + "/RETENTION");
+        Assert.assertEquals(coord.getFrequency(), "${coord:hours(12)}");
+        Assert.assertEquals(coord.getTimezone(), "UTC");
+
+        HashMap<String, String> wfProps = getWorkflowProperties(trgMiniDFS.getFileSystem(),
coord);
+        Assert.assertEquals(wfProps.get("feedNames"), lifecycleLocalRetentionFeed.getName());
+        Assert.assertTrue(StringUtils.equals(wfProps.get("entityType"), EntityType.FEED.name()));
+        Assert.assertEquals(wfProps.get("userWorkflowEngine"), "falcon");
+        Assert.assertEquals(wfProps.get("queueName"), "local");
+        Assert.assertEquals(wfProps.get("limit"), "hours(4)");
+        Assert.assertEquals(wfProps.get("jobPriority"), "HIGH");
+    }
 
     @Test
     public void testRetentionFrequency() throws Exception {

http://git-wip-us.apache.org/repos/asf/falcon/blob/1cfeea39/oozie/src/test/resources/feed/fs-local-retention-lifecycle-feed.xml
----------------------------------------------------------------------
diff --git a/oozie/src/test/resources/feed/fs-local-retention-lifecycle-feed.xml b/oozie/src/test/resources/feed/fs-local-retention-lifecycle-feed.xml
new file mode 100644
index 0000000..bdf1e59
--- /dev/null
+++ b/oozie/src/test/resources/feed/fs-local-retention-lifecycle-feed.xml
@@ -0,0 +1,61 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+<feed description="lifecycle retention feed" name="retention-local-lifecycle-test" xmlns="uri:falcon:feed:0.1">
+    <partitions>
+        <partition name="colo"/>
+        <partition name="eventTime"/>
+        <partition name="impressionHour"/>
+        <partition name="pricingModel"/>
+    </partitions>
+
+    <frequency>minutes(5)</frequency>
+    <timezone>UTC</timezone>
+    <late-arrival cut-off="minutes(1)"/>
+
+    <clusters>
+        <cluster partition="${cluster.colo}" type="source" name="corp2">
+            <validity end="2099-01-01T00:00Z" start="2012-10-01T12:00Z"/>
+            <retention action="delete" limit="days(10000)"/>
+            <lifecycle>
+                <retention-stage>
+                    <frequency>hours(12)</frequency>
+                    <queue>local</queue>
+                    <priority>HIGH</priority>
+                    <properties>
+                        <property name="retention.policy.agebaseddelete.limit" value="hours(4)"></property>
+                    </properties>
+                </retention-stage>
+            </lifecycle>
+        </cluster>
+    </clusters>
+
+    <locations>
+        <location path="/data/lifecycle/" type="data"/>
+        <location path="/data/regression/fetlrc/billing/stats" type="stats"/>
+        <location path="/data/regression/fetlrc/billing/metadata" type="meta"/>
+    </locations>
+
+    <ACL permission="0x755" group="group" owner="fetl"/>
+    <schema provider="protobuf" location="/databus/streams_local/click_rr/schema/"/>
+    <properties>
+        <property name="maxMaps" value="33" />
+        <property name="mapBandwidth" value="2" />
+    </properties>
+
+
+</feed>


Mime
View raw message