Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id ED1D010581 for ; Mon, 31 Aug 2015 22:53:59 +0000 (UTC) Received: (qmail 87947 invoked by uid 500); 31 Aug 2015 22:53:59 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 87885 invoked by uid 500); 31 Aug 2015 22:53:59 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 87876 invoked by uid 99); 31 Aug 2015 22:53:59 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 31 Aug 2015 22:53:59 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 8B57FE0426; Mon, 31 Aug 2015 22:53:59 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: vinodkv@apache.org To: common-commits@hadoop.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: MAPREDUCE-6230. Fixed RMContainerAllocator to update the new AMRMToken service name properly. Contributed by Jason Lowe (cherry picked from commit cff05bff1fe24628677d41a0d537f2c383b44faf) Date: Mon, 31 Aug 2015 22:53:59 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/branch-2.6.1 8658945b3 -> e917ef3cf MAPREDUCE-6230. Fixed RMContainerAllocator to update the new AMRMToken service name properly. Contributed by Jason Lowe (cherry picked from commit cff05bff1fe24628677d41a0d537f2c383b44faf) (cherry picked from commit 43b3b43cea1f620ce66521bcc1c4b6aec264aa9a) (cherry picked from commit 013e271036554020e3d26de4fcbb5b6970f5a9c0) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e917ef3c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e917ef3c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e917ef3c Branch: refs/heads/branch-2.6.1 Commit: e917ef3cfec1063698a88ca087dc2c5a407178ad Parents: 8658945 Author: Jian He Authored: Wed Jan 28 15:51:30 2015 -0800 Committer: Vinod Kumar Vavilapalli Committed: Mon Aug 31 15:49:24 2015 -0700 ---------------------------------------------------------------------- hadoop-mapreduce-project/CHANGES.txt | 3 + .../v2/app/rm/RMContainerAllocator.java | 5 +- .../v2/app/rm/TestRMContainerAllocator.java | 93 ++++++++++++++++++++ 3 files changed, 98 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e917ef3c/hadoop-mapreduce-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index b627e04..8a046c1 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -18,6 +18,9 @@ Release 2.6.1 - UNRELEASED MAPREDUCE-6166. Reducers do not validate checksum of map outputs when fetching directly to disk. (Eric Payne via gera) + MAPREDUCE-6230. Fixed RMContainerAllocator to update the new AMRMToken + service name properly. (Jason Lowe via jianhe) + Release 2.6.0 - 2014-11-18 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/e917ef3c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java index e60ce53..46f3f42 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java @@ -73,6 +73,7 @@ import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.Token; +import org.apache.hadoop.yarn.client.ClientRMProxy; import org.apache.hadoop.yarn.client.api.NMTokenCache; import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException; import org.apache.hadoop.yarn.exceptions.ApplicationMasterNotRegisteredException; @@ -763,10 +764,8 @@ public class RMContainerAllocator extends RMContainerRequestor .getIdentifier().array(), token.getPassword().array(), new Text( token.getKind()), new Text(token.getService())); UserGroupInformation currentUGI = UserGroupInformation.getCurrentUser(); - if (UserGroupInformation.isSecurityEnabled()) { - currentUGI = UserGroupInformation.getLoginUser(); - } currentUGI.addToken(amrmToken); + amrmToken.setService(ClientRMProxy.getAMRMTokenService(getConfig())); } @VisibleForTesting http://git-wip-us.apache.org/repos/asf/hadoop/blob/e917ef3c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java index 3a7343c..025e555 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java @@ -31,6 +31,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.io.IOException; +import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.Arrays; import java.util.EnumSet; @@ -45,6 +46,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.v2.api.records.JobId; import org.apache.hadoop.mapreduce.v2.api.records.JobState; @@ -74,7 +76,9 @@ import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.util.Time; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -109,6 +113,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.ControlledClock; @@ -2290,6 +2295,93 @@ public class TestRMContainerAllocator { } + @Test(timeout=60000) + public void testAMRMTokenUpdate() throws Exception { + LOG.info("Running testAMRMTokenUpdate"); + + final String rmAddr = "somermaddress:1234"; + final Configuration conf = new YarnConfiguration(); + conf.setLong( + YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, 8); + conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, 2000); + conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, rmAddr); + + final MyResourceManager rm = new MyResourceManager(conf); + rm.start(); + AMRMTokenSecretManager secretMgr = + rm.getRMContext().getAMRMTokenSecretManager(); + DrainDispatcher dispatcher = (DrainDispatcher) rm.getRMContext() + .getDispatcher(); + + // Submit the application + RMApp app = rm.submitApp(1024); + dispatcher.await(); + + MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); + amNodeManager.nodeHeartbeat(true); + dispatcher.await(); + + final ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt() + .getAppAttemptId(); + final ApplicationId appId = app.getApplicationId(); + rm.sendAMLaunched(appAttemptId); + dispatcher.await(); + + JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0); + final Job mockJob = mock(Job.class); + when(mockJob.getReport()).thenReturn( + MRBuilderUtils.newJobReport(jobId, "job", "user", JobState.RUNNING, 0, + 0, 0, 0, 0, 0, 0, "jobfile", null, false, "")); + + final Token oldToken = rm.getRMContext().getRMApps() + .get(appId).getRMAppAttempt(appAttemptId).getAMRMToken(); + Assert.assertNotNull("app should have a token", oldToken); + UserGroupInformation testUgi = UserGroupInformation.createUserForTesting( + "someuser", new String[0]); + Token newToken = testUgi.doAs( + new PrivilegedExceptionAction>() { + @Override + public Token run() throws Exception { + MyContainerAllocator allocator = new MyContainerAllocator(rm, conf, + appAttemptId, mockJob); + + // Keep heartbeating until RM thinks the token has been updated + Token currentToken = oldToken; + long startTime = Time.monotonicNow(); + while (currentToken == oldToken) { + if (Time.monotonicNow() - startTime > 20000) { + Assert.fail("Took to long to see AMRM token change"); + } + Thread.sleep(100); + allocator.schedule(); + currentToken = rm.getRMContext().getRMApps().get(appId) + .getRMAppAttempt(appAttemptId).getAMRMToken(); + } + + return currentToken; + } + }); + + // verify there is only one AMRM token in the UGI and it matches the + // updated token from the RM + int tokenCount = 0; + Token ugiToken = null; + for (Token token : testUgi.getTokens()) { + if (AMRMTokenIdentifier.KIND_NAME.equals(token.getKind())) { + ugiToken = token; + ++tokenCount; + } + } + + Assert.assertEquals("too many AMRM tokens", 1, tokenCount); + Assert.assertArrayEquals("token identifier not updated", + newToken.getIdentifier(), ugiToken.getIdentifier()); + Assert.assertArrayEquals("token password not updated", + newToken.getPassword(), ugiToken.getPassword()); + Assert.assertEquals("AMRM token service not updated", + new Text(rmAddr), ugiToken.getService()); + } + public static void main(String[] args) throws Exception { TestRMContainerAllocator t = new TestRMContainerAllocator(); t.testSimple(); @@ -2299,6 +2391,7 @@ public class TestRMContainerAllocator { t.testReportedAppProgressWithOnlyMaps(); t.testBlackListedNodes(); t.testCompletedTasksRecalculateSchedule(); + t.testAMRMTokenUpdate(); } }