Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 584BD200C0F for ; Thu, 29 Dec 2016 00:31:40 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 5503E160B34; Wed, 28 Dec 2016 23:31:40 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 7C1FF160B2E for ; Thu, 29 Dec 2016 00:31:39 +0100 (CET) Received: (qmail 51107 invoked by uid 500); 28 Dec 2016 23:31:38 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 51092 invoked by uid 99); 28 Dec 2016 23:31:38 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 28 Dec 2016 23:31:38 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 81F62DFC15; Wed, 28 Dec 2016 23:31:38 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: rkanter@apache.org To: common-commits@hadoop.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: YARN-4882. Change the log level to DEBUG for recovering completed applications (templedf via rkanter) Date: Wed, 28 Dec 2016 23:31:38 +0000 (UTC) archived-at: Wed, 28 Dec 2016 23:31:40 -0000 Repository: hadoop Updated Branches: refs/heads/branch-2 47f0a0e15 -> f9b48206d YARN-4882. Change the log level to DEBUG for recovering completed applications (templedf via rkanter) (cherry picked from commit f216276d2164c6564632c571fd3adbb03bc8b3e4) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f9b48206 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f9b48206 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f9b48206 Branch: refs/heads/branch-2 Commit: f9b48206ddae91835e19d965da566e5b4fc47c64 Parents: 47f0a0e Author: Robert Kanter Authored: Wed Dec 28 15:21:52 2016 -0800 Committer: Robert Kanter Committed: Wed Dec 28 15:29:06 2016 -0800 ---------------------------------------------------------------------- .../server/resourcemanager/RMAppManager.java | 13 ++++++-- .../server/resourcemanager/rmapp/RMAppImpl.java | 31 +++++++++++++++----- .../rmapp/attempt/RMAppAttemptImpl.java | 27 +++++++++++++---- 3 files changed, 57 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9b48206/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 003f1b2..d18a5f6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -480,8 +480,17 @@ public class RMAppManager implements EventHandler, Map appStates = state.getApplicationState(); LOG.info("Recovering " + appStates.size() + " applications"); - for (ApplicationStateData appState : appStates.values()) { - recoverApplication(appState, state); + + int count = 0; + + try { + for (ApplicationStateData appState : appStates.values()) { + recoverApplication(appState, state); + count += 1; + } + } finally { + LOG.info("Successfully recovered " + count + " out of " + + appStates.size() + " applications"); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9b48206/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 7b6da59..71b5ab7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -127,6 +127,10 @@ public class RMAppImpl implements RMApp, Recoverable { private static final EnumSet COMPLETED_APP_STATES = EnumSet.of(RMAppState.FINISHED, RMAppState.FINISHING, RMAppState.FAILED, RMAppState.KILLED, RMAppState.FINAL_SAVING, RMAppState.KILLING); + private static final String STATE_CHANGE_MESSAGE = + "%s State change from %s to %s on event = %s"; + private static final String RECOVERY_MESSAGE = + "Recovering app: %s with %d attempts and final state = %s"; // Immutable fields private final ApplicationId applicationId; @@ -838,9 +842,16 @@ public class RMAppImpl implements RMApp, Recoverable { /* TODO fail the application on the failed transition */ } - if (oldState != getState()) { - LOG.info(appID + " State change from " + oldState + " to " - + getState() + " on event=" + event.getType()); + // Log at INFO if we're not recovering or not in a terminal state. + // Log at DEBUG otherwise. + if ((oldState != getState()) && + (((recoveredFinalState == null)) || + (event.getType() != RMAppEventType.RECOVER))) { + LOG.info(String.format(STATE_CHANGE_MESSAGE, appID, oldState, + getState(), event.getType())); + } else if ((oldState != getState()) && LOG.isDebugEnabled()) { + LOG.debug(String.format(STATE_CHANGE_MESSAGE, appID, oldState, + getState(), event.getType())); } } finally { this.writeLock.unlock(); @@ -852,9 +863,15 @@ public class RMAppImpl implements RMApp, Recoverable { ApplicationStateData appState = state.getApplicationState().get(getApplicationId()); this.recoveredFinalState = appState.getState(); - LOG.info("Recovering app: " + getApplicationId() + " with " + - + appState.getAttemptCount() + " attempts and final state = " - + this.recoveredFinalState ); + + if (recoveredFinalState == null) { + LOG.info(String.format(RECOVERY_MESSAGE, getApplicationId(), + appState.getAttemptCount(), "NONE")); + } else if (LOG.isDebugEnabled()) { + LOG.debug(String.format(RECOVERY_MESSAGE, getApplicationId(), + appState.getAttemptCount(), recoveredFinalState)); + } + this.diagnostics.append(null == appState.getDiagnostics() ? "" : appState .getDiagnostics()); this.storedFinishTime = appState.getFinishTime(); @@ -1944,4 +1961,4 @@ public class RMAppImpl implements RMApp, Recoverable { public void setApplicationPriority(Priority applicationPriority) { this.applicationPriority = applicationPriority; } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9b48206/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 9e0e0ed..b1bc69e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -114,6 +114,10 @@ import com.google.common.annotations.VisibleForTesting; @SuppressWarnings({"unchecked", "rawtypes"}) public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { + private static final String STATE_CHANGE_MESSAGE = + "%s State change from %s to %s on event = %s"; + private static final String RECOVERY_MESSAGE = + "Recovering attempt: %s with final state = %s"; private static final Log LOG = LogFactory.getLog(RMAppAttemptImpl.class); @@ -868,9 +872,16 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { /* TODO fail the application on the failed transition */ } - if (oldState != getAppAttemptState()) { - LOG.info(appAttemptID + " State change from " + oldState + " to " - + getAppAttemptState()); + // Log at INFO if we're not recovering or not in a terminal state. + // Log at DEBUG otherwise. + if ((oldState != getAppAttemptState()) && + ((recoveredFinalState == null) || + (event.getType() != RMAppAttemptEventType.RECOVER))) { + LOG.info(String.format(STATE_CHANGE_MESSAGE, appAttemptID, oldState, + getAppAttemptState(), event.getType())); + } else if ((oldState != getAppAttemptState()) && LOG.isDebugEnabled()) { + LOG.debug(String.format(STATE_CHANGE_MESSAGE, appAttemptID, oldState, + getAppAttemptState(), event.getType())); } } finally { this.writeLock.unlock(); @@ -907,8 +918,14 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { ApplicationAttemptStateData attemptState = appState.getAttempt(getAppAttemptId()); assert attemptState != null; - LOG.info("Recovering attempt: " + getAppAttemptId() + " with final state: " - + attemptState.getState()); + + if (attemptState.getState() == null) { + LOG.info(String.format(RECOVERY_MESSAGE, getAppAttemptId(), "NONE")); + } else if (LOG.isDebugEnabled()) { + LOG.debug(String.format(RECOVERY_MESSAGE, getAppAttemptId(), + attemptState.getState())); + } + diagnostics.append("Attempt recovered after RM restart"); diagnostics.append(attemptState.getDiagnostics()); this.amContainerExitStatus = attemptState.getAMContainerExitStatus(); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org