Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 2EBD218733 for ; Fri, 22 May 2015 21:36:46 +0000 (UTC) Received: (qmail 84406 invoked by uid 500); 22 May 2015 21:36:46 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 84339 invoked by uid 500); 22 May 2015 21:36:45 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 84330 invoked by uid 99); 22 May 2015 21:36:45 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 22 May 2015 21:36:45 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id C20CEDFDDB; Fri, 22 May 2015 21:36:45 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: xgong@apache.org To: common-commits@hadoop.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: YARN-3701. Isolating the error of generating a single app report when getting all apps from generic history service. Contributed by Zhijie Shen Date: Fri, 22 May 2015 21:36:45 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/branch-2.7 c60054743 -> eb4d1ed61 YARN-3701. Isolating the error of generating a single app report when getting all apps from generic history service. Contributed by Zhijie Shen (cherry picked from commit 455b3acf0e82b214e06bd7b538968252945cd3c4) (cherry picked from commit 33be070a5e944bdc04693e7be6f39fd3d145d8d1) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eb4d1ed6 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eb4d1ed6 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eb4d1ed6 Branch: refs/heads/branch-2.7 Commit: eb4d1ed61283f497361e7cc340ac820a56ecd4a9 Parents: c600547 Author: Xuan Authored: Fri May 22 14:34:39 2015 -0700 Committer: Xuan Committed: Fri May 22 14:36:30 2015 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 ++ ...pplicationHistoryManagerOnTimelineStore.java | 37 +++++++++++++++----- ...pplicationHistoryManagerOnTimelineStore.java | 25 +++++++++---- 3 files changed, 50 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb4d1ed6/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 92f8c3f..15e6ce1 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -122,6 +122,9 @@ Release 2.7.1 - UNRELEASED YARN-3675. FairScheduler: RM quits when node removal races with continuous-scheduling on the same node. (Anubhav Dhoot via kasha) + YARN-3701. Isolating the error of generating a single app report when + getting all apps from generic history service. (Zhijie Shen via xgong) + Release 2.7.0 - 2015-04-20 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb4d1ed6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java index 49041c7..5f4510b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java @@ -25,6 +25,8 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AuthorizationException; @@ -67,6 +69,8 @@ import com.google.common.annotations.VisibleForTesting; public class ApplicationHistoryManagerOnTimelineStore extends AbstractService implements ApplicationHistoryManager { + private static final Log LOG = LogFactory + .getLog(ApplicationHistoryManagerOnTimelineStore.class); @VisibleForTesting static final String UNAVAILABLE = "N/A"; @@ -107,9 +111,14 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService new LinkedHashMap(); if (entities != null && entities.getEntities() != null) { for (TimelineEntity entity : entities.getEntities()) { - ApplicationReportExt app = - generateApplicationReport(entity, ApplicationReportField.ALL); - apps.put(app.appReport.getApplicationId(), app.appReport); + try { + ApplicationReportExt app = + generateApplicationReport(entity, ApplicationReportField.ALL); + apps.put(app.appReport.getApplicationId(), app.appReport); + } catch (Exception e) { + LOG.error("Error on generating application report for " + + entity.getEntityId(), e); + } } } return apps; @@ -142,9 +151,18 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService @Override public ApplicationAttemptReport getApplicationAttempt( ApplicationAttemptId appAttemptId) throws YarnException, IOException { - ApplicationReportExt app = getApplication( - appAttemptId.getApplicationId(), ApplicationReportField.USER_AND_ACLS); - checkAccess(app); + return getApplicationAttempt(appAttemptId, true); + } + + private ApplicationAttemptReport getApplicationAttempt( + ApplicationAttemptId appAttemptId, boolean checkACLs) + throws YarnException, IOException { + if (checkACLs) { + ApplicationReportExt app = getApplication( + appAttemptId.getApplicationId(), + ApplicationReportField.USER_AND_ACLS); + checkAccess(app); + } TimelineEntity entity = timelineDataManager.getEntity( AppAttemptMetricsConstants.ENTITY_TYPE, appAttemptId.toString(), EnumSet.allOf(Field.class), @@ -182,7 +200,8 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService @Override public ContainerReport getAMContainer(ApplicationAttemptId appAttemptId) throws YarnException, IOException { - ApplicationAttemptReport appAttempt = getApplicationAttempt(appAttemptId); + ApplicationAttemptReport appAttempt = + getApplicationAttempt(appAttemptId, false); return getContainer(appAttempt.getAMContainerId()); } @@ -515,8 +534,8 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService try { checkAccess(app); if (app.appReport.getCurrentApplicationAttemptId() != null) { - ApplicationAttemptReport appAttempt = - getApplicationAttempt(app.appReport.getCurrentApplicationAttemptId()); + ApplicationAttemptReport appAttempt = getApplicationAttempt( + app.appReport.getCurrentApplicationAttemptId(), false); app.appReport.setHost(appAttempt.getHost()); app.appReport.setRpcPort(appAttempt.getRpcPort()); app.appReport.setTrackingUrl(appAttempt.getTrackingUrl()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb4d1ed6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java index 8cf1240..fcdafc2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java @@ -78,7 +78,9 @@ public class TestApplicationHistoryManagerOnTimelineStore { store = createStore(SCALE); TimelineEntities entities = new TimelineEntities(); entities.addEntity(createApplicationTimelineEntity( - ApplicationId.newInstance(0, SCALE + 1), true, false)); + ApplicationId.newInstance(0, SCALE + 1), true, true, false)); + entities.addEntity(createApplicationTimelineEntity( + ApplicationId.newInstance(0, SCALE + 2), true, false, true)); store.put(entities); } @@ -133,9 +135,11 @@ public class TestApplicationHistoryManagerOnTimelineStore { TimelineEntities entities = new TimelineEntities(); ApplicationId appId = ApplicationId.newInstance(0, i); if (i == 2) { - entities.addEntity(createApplicationTimelineEntity(appId, true, true)); + entities.addEntity(createApplicationTimelineEntity( + appId, true, false, false)); } else { - entities.addEntity(createApplicationTimelineEntity(appId, false, true)); + entities.addEntity(createApplicationTimelineEntity( + appId, false, false, false)); } store.put(entities); for (int j = 1; j <= scale; ++j) { @@ -334,6 +338,10 @@ public class TestApplicationHistoryManagerOnTimelineStore { historyManager.getAllApplications().values(); Assert.assertNotNull(apps); Assert.assertEquals(SCALE + 1, apps.size()); + ApplicationId ignoredAppId = ApplicationId.newInstance(0, SCALE + 2); + for (ApplicationReport app : apps) { + Assert.assertNotEquals(ignoredAppId, app.getApplicationId()); + } } @Test @@ -433,10 +441,15 @@ public class TestApplicationHistoryManagerOnTimelineStore { } private static TimelineEntity createApplicationTimelineEntity( - ApplicationId appId, boolean emptyACLs, boolean noAttempt) { + ApplicationId appId, boolean emptyACLs, boolean noAttemptId, + boolean wrongAppId) { TimelineEntity entity = new TimelineEntity(); entity.setEntityType(ApplicationMetricsConstants.ENTITY_TYPE); - entity.setEntityId(appId.toString()); + if (wrongAppId) { + entity.setEntityId("wrong_app_id"); + } else { + entity.setEntityId(appId.toString()); + } entity.setDomainId(TimelineDataManager.DEFAULT_DOMAIN_ID); entity.addPrimaryFilter( TimelineStore.SystemFilter.ENTITY_OWNER.toString(), "yarn"); @@ -472,7 +485,7 @@ public class TestApplicationHistoryManagerOnTimelineStore { FinalApplicationStatus.UNDEFINED.toString()); eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO, YarnApplicationState.FINISHED.toString()); - if (noAttempt) { + if (!noAttemptId) { eventInfo.put(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO, ApplicationAttemptId.newInstance(appId, 1)); }