From common-commits-return-86278-archive-asf-public=cust-asf.ponee.io@hadoop.apache.org Wed Aug 1 00:24:45 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id B69AD180662 for ; Wed, 1 Aug 2018 00:24:44 +0200 (CEST) Received: (qmail 30746 invoked by uid 500); 31 Jul 2018 22:24:43 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 30737 invoked by uid 99); 31 Jul 2018 22:24:43 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 31 Jul 2018 22:24:43 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 96EBBDFC65; Tue, 31 Jul 2018 22:24:43 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: eyang@apache.org To: common-commits@hadoop.apache.org Message-Id: <46dbf40d2f3a4e3ab07ae89935005dc7@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: YARN-8579. Recover NMToken of previous attempted component data. Contributed by Gour Saha Date: Tue, 31 Jul 2018 22:24:43 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/trunk 4b540bbfc -> c7ebcd76b YARN-8579. Recover NMToken of previous attempted component data. Contributed by Gour Saha Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c7ebcd76 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c7ebcd76 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c7ebcd76 Branch: refs/heads/trunk Commit: c7ebcd76bf3dd14127336951f2be3de772e7826a Parents: 4b540bb Author: Eric Yang Authored: Tue Jul 31 18:01:02 2018 -0400 Committer: Eric Yang Committed: Tue Jul 31 18:01:02 2018 -0400 ---------------------------------------------------------------------- .../hadoop/yarn/service/ServiceScheduler.java | 1 + .../scheduler/SchedulerApplicationAttempt.java | 3 ++- .../scheduler/fair/FairScheduler.java | 8 ++++++-- .../applicationsmanager/TestAMRestart.java | 18 ++++++++++++++---- 4 files changed, 23 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7ebcd76/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java index cfaf356..0801ad0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java @@ -649,6 +649,7 @@ public class ServiceScheduler extends CompositeService { @Override public void onContainersReceivedFromPreviousAttempts( List containers) { + LOG.info("Containers recovered after AM registered: {}", containers); if (containers == null || containers.isEmpty()) { return; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7ebcd76/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index dd6d38f..f9df2b8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -785,6 +785,7 @@ public class SchedulerApplicationAttempt implements SchedulableEntity { List returnContainerList = new ArrayList<> (recoveredPreviousAttemptContainers); recoveredPreviousAttemptContainers.clear(); + updateNMTokens(returnContainerList); return returnContainerList; } finally { writeLock.unlock(); @@ -1466,4 +1467,4 @@ public class SchedulerApplicationAttempt implements SchedulableEntity { public Map getApplicationSchedulingEnvs() { return this.applicationSchedulingEnvs; } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7ebcd76/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 20d1afe..037cebf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.NMToken; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.QueueInfo; @@ -950,12 +951,15 @@ public class FairScheduler extends Resource headroom = application.getHeadroom(); application.setApplicationHeadroomForMetrics(headroom); + List previousAttemptContainers = application + .pullPreviousAttemptContainers(); + List updatedNMTokens = application.pullUpdatedNMTokens(); return new Allocation(newlyAllocatedContainers, headroom, preemptionContainerIds, null, null, - application.pullUpdatedNMTokens(), null, null, + updatedNMTokens, null, null, application.pullNewlyPromotedContainers(), application.pullNewlyDemotedContainers(), - application.pullPreviousAttemptContainers()); + previousAttemptContainers); } private List validateResourceRequests( http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7ebcd76/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java index 4add186..9f122cb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java @@ -1048,12 +1048,12 @@ public class TestAMRestart extends ParameterizedSchedulerTestBase { rm1.start(); YarnScheduler scheduler = rm1.getResourceScheduler(); - MockNM nm1 = new MockNM("127.0.0.1:1234", 10240, - rm1.getResourceTrackerService()); + String nm1Address = "127.0.0.1:1234"; + MockNM nm1 = new MockNM(nm1Address, 10240, rm1.getResourceTrackerService()); nm1.registerNode(); - MockNM nm2 = new MockNM("127.0.0.1:2351", 4089, - rm1.getResourceTrackerService()); + String nm2Address = "127.0.0.1:2351"; + MockNM nm2 = new MockNM(nm2Address, 4089, rm1.getResourceTrackerService()); nm2.registerNode(); RMApp app1 = rm1.submitApp(200, "name", "user", @@ -1120,6 +1120,11 @@ public class TestAMRestart extends ParameterizedSchedulerTestBase { registerResponse.getContainersFromPreviousAttempts().size()); Assert.assertEquals("container 2", containerId2, registerResponse.getContainersFromPreviousAttempts().get(0).getId()); + List prevNMTokens = registerResponse + .getNMTokensFromPreviousAttempts(); + Assert.assertEquals(1, prevNMTokens.size()); + // container 2 is running on node 1 + Assert.assertEquals(nm1Address, prevNMTokens.get(0).getNodeId().toString()); rm2.waitForState(app1.getApplicationId(), RMAppState.RUNNING); @@ -1145,6 +1150,11 @@ public class TestAMRestart extends ParameterizedSchedulerTestBase { allocateResponse.getContainersFromPreviousAttempts()); Assert.assertEquals("new containers should not be allocated", 0, allocateResponse.getAllocatedContainers().size()); + List nmTokens = allocateResponse.getNMTokens(); + Assert.assertEquals(1, nmTokens.size()); + // container 3 is running on node 2 + Assert.assertEquals(nm2Address, + nmTokens.get(0).getNodeId().toString()); return true; } } catch (Exception e) { --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org