Return-Path: X-Original-To: apmail-falcon-commits-archive@minotaur.apache.org Delivered-To: apmail-falcon-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 1156518706 for ; Wed, 6 Jan 2016 10:40:07 +0000 (UTC) Received: (qmail 52264 invoked by uid 500); 6 Jan 2016 10:40:07 -0000 Delivered-To: apmail-falcon-commits-archive@falcon.apache.org Received: (qmail 52189 invoked by uid 500); 6 Jan 2016 10:40:07 -0000 Mailing-List: contact commits-help@falcon.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@falcon.apache.org Delivered-To: mailing list commits@falcon.apache.org Received: (qmail 52172 invoked by uid 99); 6 Jan 2016 10:40:06 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 06 Jan 2016 10:40:06 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id B5455DFBE6; Wed, 6 Jan 2016 10:40:06 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: pallavi@apache.org To: commits@falcon.apache.org Date: Wed, 06 Jan 2016 10:40:07 -0000 Message-Id: <0decc2b808834fb6af978b6a8f1a2465@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [2/2] falcon git commit: FALCON-1720 Rerun API does not rerun succeeded instances (by Pavan Kolamuri) FALCON-1720 Rerun API does not rerun succeeded instances (by Pavan Kolamuri) Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/ccb6df38 Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/ccb6df38 Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/ccb6df38 Branch: refs/heads/master Commit: ccb6df38bf086135d28bce9c0c92d6fd23e19459 Parents: 7cde36c Author: Pallavi Rao Authored: Wed Jan 6 15:35:31 2016 +0530 Committer: Pallavi Rao Committed: Wed Jan 6 15:35:31 2016 +0530 ---------------------------------------------------------------------- CHANGES.txt | 2 ++ .../falcon/workflow/engine/OozieDAGEngine.java | 24 +++++++++++++++----- .../AbstractSchedulerManagerJerseyIT.java | 2 +- .../InstanceSchedulerManagerJerseyIT.java | 12 +++++++++- .../local-process-noinputs-template.xml | 2 +- 5 files changed, 33 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/ccb6df38/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 2ed1ab4..5677175 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -90,6 +90,8 @@ Proposed Release Version: 0.9 OPTIMIZATIONS BUG FIXES + FALCON-1720 Rerun API does not rerun succeeded instances (Pavan Kolamuri via Pallavi Rao) + FALCON-1719 Retry does not update the state of the instance in the database (Pavan Kolamuri via Pallavi Rao) FALCON-1710 dependency API sets totalResults as 0 by default(Praveen Adlakha via Ajay Yadava) http://git-wip-us.apache.org/repos/asf/falcon/blob/ccb6df38/scheduler/src/main/java/org/apache/falcon/workflow/engine/OozieDAGEngine.java ---------------------------------------------------------------------- diff --git a/scheduler/src/main/java/org/apache/falcon/workflow/engine/OozieDAGEngine.java b/scheduler/src/main/java/org/apache/falcon/workflow/engine/OozieDAGEngine.java index 1425a97..1d0e126 100644 --- a/scheduler/src/main/java/org/apache/falcon/workflow/engine/OozieDAGEngine.java +++ b/scheduler/src/main/java/org/apache/falcon/workflow/engine/OozieDAGEngine.java @@ -237,17 +237,29 @@ public class OozieDAGEngine implements DAGEngine { String jobId = instance.getExternalID(); try { WorkflowJob jobInfo = client.getJobInfo(jobId); - Properties jobprops = OozieUtils.toProperties(jobInfo.getConf()); - if (props != null) { - jobprops.putAll(props); + if (props == null) { + props = new Properties(); } //if user has set any of these oozie rerun properties then force rerun flag is ignored - if (!jobprops.containsKey(OozieClient.RERUN_FAIL_NODES) - && !jobprops.containsKey(OozieClient.RERUN_SKIP_NODES)) { - jobprops.put(OozieClient.RERUN_FAIL_NODES, String.valueOf(!isForced)); + if (!props.containsKey(OozieClient.RERUN_FAIL_NODES) + && !props.containsKey(OozieClient.RERUN_SKIP_NODES)) { + props.put(OozieClient.RERUN_FAIL_NODES, String.valueOf(!isForced)); } + + Properties jobprops = OozieUtils.toProperties(jobInfo.getConf()); + jobprops.putAll(props); + jobprops.remove(OozieClient.COORDINATOR_APP_PATH); jobprops.remove(OozieClient.BUNDLE_APP_PATH); + // In case if both props exists one should be removed otherwise it will fail. + // This case will occur when user runs workflow with skip-nodes property and + // try to do force rerun or rerun with fail-nodes property. + if (jobprops.containsKey(OozieClient.RERUN_FAIL_NODES) + && jobprops.containsKey(OozieClient.RERUN_SKIP_NODES)) { + LOG.warn("Both " + OozieClient.RERUN_SKIP_NODES + " and " + OozieClient.RERUN_FAIL_NODES + + " are present in workflow params removing" + OozieClient.RERUN_SKIP_NODES); + jobprops.remove(OozieClient.RERUN_SKIP_NODES); + } client.reRun(jobId, jobprops); assertStatus(instance.getExternalID(), Job.Status.PREP, Job.Status.RUNNING, Job.Status.SUCCEEDED); LOG.info("Rerun job {} of entity {} of time {} on cluster {}", jobId, instance.getEntity().getName(), http://git-wip-us.apache.org/repos/asf/falcon/blob/ccb6df38/webapp/src/test/java/org/apache/falcon/resource/AbstractSchedulerManagerJerseyIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/falcon/resource/AbstractSchedulerManagerJerseyIT.java b/webapp/src/test/java/org/apache/falcon/resource/AbstractSchedulerManagerJerseyIT.java index f053b76..0a3e984 100644 --- a/webapp/src/test/java/org/apache/falcon/resource/AbstractSchedulerManagerJerseyIT.java +++ b/webapp/src/test/java/org/apache/falcon/resource/AbstractSchedulerManagerJerseyIT.java @@ -62,7 +62,7 @@ public class AbstractSchedulerManagerJerseyIT extends FalconUnitTestBase { private static final String DB_BASE_DIR = "target/test-data/falcondb"; protected static String dbLocation = DB_BASE_DIR + File.separator + "data.db"; protected static String url = "jdbc:derby:"+ dbLocation +";create=true"; - protected static final String DB_SQL_FILE = DB_BASE_DIR + File.separator + "out.sql"; + protected static final String DB_SQL_FILE = dbLocation + File.separator + "out.sql"; protected LocalFileSystem localFS = new LocalFileSystem(); http://git-wip-us.apache.org/repos/asf/falcon/blob/ccb6df38/webapp/src/test/java/org/apache/falcon/resource/InstanceSchedulerManagerJerseyIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/falcon/resource/InstanceSchedulerManagerJerseyIT.java b/webapp/src/test/java/org/apache/falcon/resource/InstanceSchedulerManagerJerseyIT.java index 1523b76..18c36ff 100644 --- a/webapp/src/test/java/org/apache/falcon/resource/InstanceSchedulerManagerJerseyIT.java +++ b/webapp/src/test/java/org/apache/falcon/resource/InstanceSchedulerManagerJerseyIT.java @@ -64,7 +64,7 @@ public class InstanceSchedulerManagerJerseyIT extends AbstractSchedulerManagerJe } @Test - public void testKillInstances() throws Exception { + public void testKillAndRerunInstances() throws Exception { UnitTestContext context = new UnitTestContext(); Map overlay = context.getUniqueOverlay(); @@ -84,6 +84,16 @@ public class InstanceSchedulerManagerJerseyIT extends AbstractSchedulerManagerJe processName, START_INSTANCE); Assert.assertEquals(status, InstancesResult.WorkflowStatus.KILLED); + result = falconUnitClient.rerunInstances(EntityType.PROCESS.toString(), + processName, START_INSTANCE, END_TIME, colo, null, null, null, null, true, null); + assertStatus(result); + + waitForStatus(EntityType.PROCESS.name(), context.processName, START_INSTANCE, + InstancesResult.WorkflowStatus.RUNNING); + status = getClient().getInstanceStatus(EntityType.PROCESS.name(), + processName, START_INSTANCE); + Assert.assertEquals(status, InstancesResult.WorkflowStatus.RUNNING); + } http://git-wip-us.apache.org/repos/asf/falcon/blob/ccb6df38/webapp/src/test/resources/local-process-noinputs-template.xml ---------------------------------------------------------------------- diff --git a/webapp/src/test/resources/local-process-noinputs-template.xml b/webapp/src/test/resources/local-process-noinputs-template.xml index aabdc6a..a7388d8 100644 --- a/webapp/src/test/resources/local-process-noinputs-template.xml +++ b/webapp/src/test/resources/local-process-noinputs-template.xml @@ -38,5 +38,5 @@ - +