Return-Path: X-Original-To: apmail-tez-commits-archive@minotaur.apache.org Delivered-To: apmail-tez-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 4A2F918960 for ; Fri, 21 Aug 2015 01:36:10 +0000 (UTC) Received: (qmail 99074 invoked by uid 500); 21 Aug 2015 01:36:10 -0000 Delivered-To: apmail-tez-commits-archive@tez.apache.org Received: (qmail 98986 invoked by uid 500); 21 Aug 2015 01:36:10 -0000 Mailing-List: contact commits-help@tez.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@tez.apache.org Delivered-To: mailing list commits@tez.apache.org Received: (qmail 97708 invoked by uid 99); 21 Aug 2015 01:36:09 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 21 Aug 2015 01:36:09 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 52AEEE7DF6; Fri, 21 Aug 2015 01:36:09 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: sseth@apache.org To: commits@tez.apache.org Date: Fri, 21 Aug 2015 01:36:29 -0000 Message-Id: In-Reply-To: <0a6214bdcf644e979ab2906bb3bbf947@git.apache.org> References: <0a6214bdcf644e979ab2906bb3bbf947@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [21/50] [abbrv] tez git commit: TEZ-2443. TaskRunner2 should call abort, NPEs while cleaning up tasks. (sseth) TEZ-2443. TaskRunner2 should call abort, NPEs while cleaning up tasks. (sseth) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/848b137f Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/848b137f Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/848b137f Branch: refs/heads/TEZ-2003 Commit: 848b137f43af982dae9c9a51594170c13d59fb91 Parents: 12a3729 Author: Siddharth Seth Authored: Tue May 12 14:27:42 2015 -0700 Committer: Siddharth Seth Committed: Thu Aug 20 18:22:07 2015 -0700 ---------------------------------------------------------------------- TEZ-2003-CHANGES.txt | 1 + .../java/org/apache/tez/dag/api/TaskCommunicator.java | 4 ++++ .../tez/runtime/LogicalIOProcessorRuntimeTask.java | 11 ++++++----- .../main/java/org/apache/tez/runtime/RuntimeTask.java | 2 +- .../apache/tez/runtime/task/TaskRunner2Callable.java | 13 +++++++------ .../org/apache/tez/runtime/task/TezTaskRunner2.java | 10 ++++++---- 6 files changed, 25 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/848b137f/TEZ-2003-CHANGES.txt ---------------------------------------------------------------------- diff --git a/TEZ-2003-CHANGES.txt b/TEZ-2003-CHANGES.txt index 5d2e40a..ed72d6b 100644 --- a/TEZ-2003-CHANGES.txt +++ b/TEZ-2003-CHANGES.txt @@ -25,5 +25,6 @@ ALL CHANGES: TEZ-2433. Fixes after rebase 05/08 TEZ-2438. tez-tools version in the branch is incorrect. TEZ-2434. Allow tasks to be killed in the Runtime. + TEZ-2443. TaskRunner2 should call abort, NPEs while cleaning up tasks. INCOMPATIBLE CHANGES: http://git-wip-us.apache.org/repos/asf/tez/blob/848b137f/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicator.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicator.java b/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicator.java index cadca0c..2651013 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicator.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicator.java @@ -48,6 +48,10 @@ public abstract class TaskCommunicator extends AbstractService { // TODO TEZ-2003. Are additional APIs required to mark a container as completed ? - for completeness. // TODO TEZ-2003 Remove reference to TaskAttemptID + // TODO TEZ-2003 This needs some information about why the attempt is being unregistered. + // e.g. preempted in which case the task may need to be informed. Alternately as a result of + // a failed task. + // In case of preemption - a killTask API is likely a better bet than trying to overload this method. public abstract void unregisterRunningTaskAttempt(TezTaskAttemptID taskAttemptID); // TODO TEZ-2003 This doesn't necessarily belong here. A server may not start within the AM. http://git-wip-us.apache.org/repos/asf/tez/blob/848b137f/tez-runtime-internals/src/main/java/org/apache/tez/runtime/LogicalIOProcessorRuntimeTask.java ---------------------------------------------------------------------- diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/LogicalIOProcessorRuntimeTask.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/LogicalIOProcessorRuntimeTask.java index 8263b3f..de08e56 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/LogicalIOProcessorRuntimeTask.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/LogicalIOProcessorRuntimeTask.java @@ -704,7 +704,7 @@ public class LogicalIOProcessorRuntimeTask extends RuntimeTask { } @Override - public synchronized void abortTask() throws Exception { + public synchronized void abortTask() { if (processor != null) { processor.abort(); } @@ -803,6 +803,7 @@ public class LogicalIOProcessorRuntimeTask extends RuntimeTask { LOG.debug("Num of inputs to be closed={}", initializedInputs.size()); LOG.debug("Num of outputs to be closed={}", initializedOutputs.size()); } + // Close processor if (!processorClosed && processor != null) { try { @@ -820,8 +821,8 @@ public class LogicalIOProcessorRuntimeTask extends RuntimeTask { Thread.currentThread().interrupt(); } catch (Throwable e) { LOG.warn( - "Ignoring Exception when closing processor(cleanup). Exception class={}, message={}", - e.getClass().getName(), e.getMessage()); + "Ignoring Exception when closing processor(cleanup). Exception class={}, message={}" + + e.getClass().getName(), e.getMessage(), e); } } @@ -842,7 +843,7 @@ public class LogicalIOProcessorRuntimeTask extends RuntimeTask { } catch (Throwable e) { LOG.warn( "Ignoring exception when closing input {}(cleanup). Exception class={}, message={}", - srcVertexName, e.getClass().getName(), e.getMessage()); + srcVertexName, e.getClass().getName(), e.getMessage(), e); } finally { LOG.info("Close input for vertex={}, sourceVertex={}, interruptedStatus={}", processor .getContext().getTaskVertexName(), srcVertexName, Thread.currentThread().isInterrupted()); @@ -866,7 +867,7 @@ public class LogicalIOProcessorRuntimeTask extends RuntimeTask { } catch (Throwable e) { LOG.warn( "Ignoring exception when closing output {}(cleanup). Exception class={}, message={}", - destVertexName, e.getClass().getName(), e.getMessage()); + destVertexName, e.getClass().getName(), e.getMessage(), e); } finally { LOG.info("Close input for vertex={}, sourceVertex={}, interruptedStatus={}", processor .getContext().getTaskVertexName(), destVertexName, Thread.currentThread().isInterrupted()); http://git-wip-us.apache.org/repos/asf/tez/blob/848b137f/tez-runtime-internals/src/main/java/org/apache/tez/runtime/RuntimeTask.java ---------------------------------------------------------------------- diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/RuntimeTask.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/RuntimeTask.java index cdfb46a..33c0113 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/RuntimeTask.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/RuntimeTask.java @@ -167,5 +167,5 @@ public abstract class RuntimeTask { taskDone.set(true); } - public abstract void abortTask() throws Exception; + public abstract void abortTask(); } http://git-wip-us.apache.org/repos/asf/tez/blob/848b137f/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TaskRunner2Callable.java ---------------------------------------------------------------------- diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TaskRunner2Callable.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TaskRunner2Callable.java index 7315bbd..ab77635 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TaskRunner2Callable.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TaskRunner2Callable.java @@ -63,26 +63,26 @@ public class TaskRunner2Callable extends CallableWithNdc