falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ajayyad...@apache.org
Subject falcon git commit: FALCON-1499 Lifecycle Retention Issues. Contributed by Ajay Yadava.
Date Thu, 01 Oct 2015 10:44:10 GMT
Repository: falcon
Updated Branches:
  refs/heads/master 8e0d5c584 -> 4783a19c4


FALCON-1499 Lifecycle Retention Issues. Contributed by Ajay Yadava.


Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/4783a19c
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/4783a19c
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/4783a19c

Branch: refs/heads/master
Commit: 4783a19c4cfbb673a647b4382961fb0df9b034a6
Parents: 8e0d5c5
Author: Ajay Yadava <ajaynsit@gmail.com>
Authored: Thu Oct 1 12:28:34 2015 +0530
Committer: Ajay Yadava <ajaynsit@gmail.com>
Committed: Thu Oct 1 13:48:55 2015 +0530

----------------------------------------------------------------------
 CHANGES.txt                                                 | 2 ++
 common/src/main/resources/startup.properties                | 4 +++-
 .../engine/oozie/retention/AgeBasedCoordinatorBuilder.java  | 2 ++
 .../engine/oozie/retention/AgeBasedWorkflowBuilder.java     | 9 +++++++--
 src/conf/startup.properties                                 | 4 +++-
 5 files changed, 17 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/falcon/blob/4783a19c/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 22aa5ff..b2104f4 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -47,6 +47,8 @@ Trunk (Unreleased)
     FALCON-1403 Revisit IT cleanup and teardown(Narayan Periwal via Pallavi Rao)
 
   BUG FIXES
+    FALCON-1499 Lifecycle Retention Issues(Ajay Yadava)
+
     FALCON-1469 Flaky EntityManagerJerseyIT#testDuplicateSubmitCommands(Sandeep Samudrala
via Ajay Yadava)
 
     FALCON-1470 HiveDRStatusStoreTest should fail when using fakeGroup to create StatusStore(Balu
Vellanki via Sowmya Ramesh)

http://git-wip-us.apache.org/repos/asf/falcon/blob/4783a19c/common/src/main/resources/startup.properties
----------------------------------------------------------------------
diff --git a/common/src/main/resources/startup.properties b/common/src/main/resources/startup.properties
index 3383129..e853c5a 100644
--- a/common/src/main/resources/startup.properties
+++ b/common/src/main/resources/startup.properties
@@ -34,6 +34,8 @@
 *.application.services=org.apache.falcon.security.AuthenticationInitializationService,\
                         org.apache.falcon.workflow.WorkflowJobEndNotificationService, \
                         org.apache.falcon.service.ProcessSubscriberService,\
+                        org.apache.falcon.service.FeedSLAMonitoringService,\
+                        org.apache.falcon.service.LifecyclePolicyMap,\
                         org.apache.falcon.entity.store.ConfigurationStore,\
                         org.apache.falcon.rerun.service.RetryService,\
                         org.apache.falcon.rerun.service.LateRunService,\
@@ -76,7 +78,7 @@
 #Configurations used in UTs
 debug.config.store.uri=file://${user.dir}/target/store
 #Location to store state of Feed SLA monitoring service
-debug.pending.feed.instances.store.uri = file://${user.dir}/data/sla/pendingfeedinstances
+debug.feed.sla.service.store.uri= file://${user.dir}/target/data/sla/pendingfeedinstances
 debug.config.oozie.conf.uri=${user.dir}/target/oozie
 debug.system.lib.location=${system.lib.location}
 debug.broker.url=vm://localhost

http://git-wip-us.apache.org/repos/asf/falcon/blob/4783a19c/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/retention/AgeBasedCoordinatorBuilder.java
----------------------------------------------------------------------
diff --git a/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/retention/AgeBasedCoordinatorBuilder.java
b/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/retention/AgeBasedCoordinatorBuilder.java
index 4601070..e3e7fec 100644
--- a/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/retention/AgeBasedCoordinatorBuilder.java
+++ b/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/retention/AgeBasedCoordinatorBuilder.java
@@ -83,6 +83,8 @@ public final class AgeBasedCoordinatorBuilder {
         CONTROLS controls = new CONTROLS();
         controls.setExecution(ExecutionType.LAST_ONLY.value());
         controls.setTimeout(String.valueOf(frequencyInMillis / (1000 * 60)));
+        controls.setConcurrency("1");
+        controls.setThrottle("1");
         coord.setControls(controls);
 
         coord.setFrequency("${coord:" + retentionFrequency.toString() + "}");

http://git-wip-us.apache.org/repos/asf/falcon/blob/4783a19c/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/retention/AgeBasedWorkflowBuilder.java
----------------------------------------------------------------------
diff --git a/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/retention/AgeBasedWorkflowBuilder.java
b/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/retention/AgeBasedWorkflowBuilder.java
index 0a87213..34b186e 100644
--- a/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/retention/AgeBasedWorkflowBuilder.java
+++ b/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/retention/AgeBasedWorkflowBuilder.java
@@ -18,6 +18,7 @@
 
 package org.apache.falcon.lifecycle.engine.oozie.retention;
 
+import org.apache.commons.lang3.StringUtils;
 import org.apache.falcon.FalconException;
 import org.apache.falcon.LifeCycle;
 import org.apache.falcon.Tag;
@@ -82,8 +83,12 @@ public final class AgeBasedWorkflowBuilder {
         props.putAll(FeedHelper.getUserWorkflowProperties(LifeCycle.EVICTION));
         // override the queueName and priority
         RetentionStage retentionStage = FeedHelper.getRetentionStage(feed, cluster.getName());
-        props.put(OozieBuilderUtils.MR_QUEUE_NAME, retentionStage.getQueue());
-        props.put(OozieBuilderUtils.MR_JOB_PRIORITY, retentionStage.getPriority());
+        if (StringUtils.isNotBlank(retentionStage.getQueue())) {
+            props.put(OozieBuilderUtils.MR_QUEUE_NAME, retentionStage.getQueue());
+        }
+        if (StringUtils.isNotBlank(retentionStage.getPriority())) {
+            props.put(OozieBuilderUtils.MR_JOB_PRIORITY, retentionStage.getPriority());
+        }
 
         if (EntityUtil.isTableStorageType(cluster, feed)) {
             setupHiveCredentials(cluster, buildPath, workflow);

http://git-wip-us.apache.org/repos/asf/falcon/blob/4783a19c/src/conf/startup.properties
----------------------------------------------------------------------
diff --git a/src/conf/startup.properties b/src/conf/startup.properties
index 8e4ce97..37cb044 100644
--- a/src/conf/startup.properties
+++ b/src/conf/startup.properties
@@ -39,6 +39,7 @@
                         org.apache.falcon.workflow.WorkflowJobEndNotificationService, \
                         org.apache.falcon.service.ProcessSubscriberService,\
                         org.apache.falcon.service.FeedSLAMonitoringService,\
+                        org.apache.falcon.service.LifecyclePolicyMap,\
                         org.apache.falcon.entity.store.ConfigurationStore,\
                         org.apache.falcon.rerun.service.RetryService,\
                         org.apache.falcon.rerun.service.LateRunService,\
@@ -48,7 +49,8 @@
                         org.apache.falcon.service.ProxyUserService
 
 ##### Prism Services #####
-prism.application.services=org.apache.falcon.entity.store.ConfigurationStore
+prism.application.services=org.apache.falcon.service.LifecyclePolicyMap,\
+                        org.apache.falcon.entity.store.ConfigurationStore
 
 
 # List of Lifecycle policies configured.


Mime
View raw message