Return-Path: X-Original-To: apmail-hadoop-yarn-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-yarn-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 8F46A10605 for ; Tue, 3 Sep 2013 23:21:34 +0000 (UTC) Received: (qmail 48802 invoked by uid 500); 3 Sep 2013 23:21:34 -0000 Delivered-To: apmail-hadoop-yarn-commits-archive@hadoop.apache.org Received: (qmail 48777 invoked by uid 500); 3 Sep 2013 23:21:34 -0000 Mailing-List: contact yarn-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: yarn-commits@hadoop.apache.org Delivered-To: mailing list yarn-commits@hadoop.apache.org Received: (qmail 48769 invoked by uid 99); 3 Sep 2013 23:21:34 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 03 Sep 2013 23:21:34 +0000 X-ASF-Spam-Status: No, hits=-1999.6 required=5.0 tests=ALL_TRUSTED,FILL_THIS_FORM_FRAUD_PHISH,T_FILL_THIS_FORM_SHORT X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 03 Sep 2013 23:21:30 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id D7A242388B34; Tue, 3 Sep 2013 23:20:50 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1519885 [3/3] - in /hadoop/common/branches/HDFS-2832/hadoop-yarn-project: ./ hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/ hadoop-yarn/... Date: Tue, 03 Sep 2013 23:20:48 -0000 To: yarn-commits@hadoop.apache.org From: arp@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20130903232050.D7A242388B34@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java?rev=1519885&r1=1519884&r2=1519885&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java Tue Sep 3 23:20:45 2013 @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.recovery; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @@ -25,12 +26,12 @@ import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; -import java.util.List; import java.util.Map; +import javax.crypto.SecretKey; + import junit.framework.Assert; import org.apache.commons.logging.Log; @@ -55,7 +56,6 @@ import org.apache.hadoop.yarn.conf.YarnC import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; -import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier; import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationState; @@ -198,7 +198,7 @@ public class TestRMStateStore { ContainerId storeAttempt(RMStateStore store, ApplicationAttemptId attemptId, String containerIdStr, Token appToken, - Token clientToAMToken, TestDispatcher dispatcher) + SecretKey clientTokenMasterKey, TestDispatcher dispatcher) throws Exception { Container container = new ContainerPBImpl(); @@ -207,7 +207,8 @@ public class TestRMStateStore { when(mockAttempt.getAppAttemptId()).thenReturn(attemptId); when(mockAttempt.getMasterContainer()).thenReturn(container); when(mockAttempt.getAMRMToken()).thenReturn(appToken); - when(mockAttempt.getClientToAMToken()).thenReturn(clientToAMToken); + when(mockAttempt.getClientTokenMasterKey()) + .thenReturn(clientTokenMasterKey); dispatcher.attemptId = attemptId; dispatcher.storedException = null; store.storeApplicationAttempt(mockAttempt); @@ -215,7 +216,6 @@ public class TestRMStateStore { return container.getId(); } - @SuppressWarnings("unchecked") void testRMAppStateStore(RMStateStoreHelper stateStoreHelper) throws Exception { long submitTime = System.currentTimeMillis(); Configuration conf = new YarnConfiguration(); @@ -233,33 +233,33 @@ public class TestRMStateStore { ApplicationId appId1 = attemptId1.getApplicationId(); storeApp(store, appId1, submitTime); - // create application token1 for attempt1 - List> appAttemptToken1 = - generateTokens(attemptId1, appTokenMgr, clientToAMTokenMgr, conf); + // create application token and client token key for attempt1 + Token appAttemptToken1 = + generateAMRMToken(attemptId1, appTokenMgr); HashSet> attemptTokenSet1 = new HashSet>(); - attemptTokenSet1.addAll(appAttemptToken1); + attemptTokenSet1.add(appAttemptToken1); + SecretKey clientTokenKey1 = + clientToAMTokenMgr.registerApplication(attemptId1); ContainerId containerId1 = storeAttempt(store, attemptId1, "container_1352994193343_0001_01_000001", - (Token) (appAttemptToken1.get(0)), - (Token)(appAttemptToken1.get(1)), - dispatcher); + appAttemptToken1, clientTokenKey1, dispatcher); String appAttemptIdStr2 = "appattempt_1352994193343_0001_000002"; ApplicationAttemptId attemptId2 = ConverterUtils.toApplicationAttemptId(appAttemptIdStr2); - // create application token2 for attempt2 - List> appAttemptToken2 = - generateTokens(attemptId2, appTokenMgr, clientToAMTokenMgr, conf); + // create application token and client token key for attempt2 + Token appAttemptToken2 = + generateAMRMToken(attemptId2, appTokenMgr); HashSet> attemptTokenSet2 = new HashSet>(); - attemptTokenSet2.addAll(appAttemptToken2); + attemptTokenSet2.add(appAttemptToken2); + SecretKey clientTokenKey2 = + clientToAMTokenMgr.registerApplication(attemptId2); ContainerId containerId2 = storeAttempt(store, attemptId2, "container_1352994193343_0001_02_000001", - (Token) (appAttemptToken2.get(0)), - (Token)(appAttemptToken2.get(1)), - dispatcher); + appAttemptToken2, clientTokenKey2, dispatcher); ApplicationAttemptId attemptIdRemoved = ConverterUtils .toApplicationAttemptId("appattempt_1352994193343_0002_000001"); @@ -306,8 +306,12 @@ public class TestRMStateStore { assertEquals(containerId1, attemptState.getMasterContainer().getId()); // attempt1 applicationToken is loaded correctly HashSet> savedTokens = new HashSet>(); - savedTokens.addAll(attemptState.getAppAttemptTokens().getAllTokens()); + savedTokens.addAll(attemptState.getAppAttemptCredentials().getAllTokens()); assertEquals(attemptTokenSet1, savedTokens); + // attempt1 client token master key is loaded correctly + assertArrayEquals(clientTokenKey1.getEncoded(), + attemptState.getAppAttemptCredentials() + .getSecretKey(RMStateStore.AM_CLIENT_TOKEN_MASTER_KEY_NAME)); attemptState = appState.getAttempt(attemptId2); // attempt2 is loaded correctly @@ -317,8 +321,12 @@ public class TestRMStateStore { assertEquals(containerId2, attemptState.getMasterContainer().getId()); // attempt2 applicationToken is loaded correctly savedTokens.clear(); - savedTokens.addAll(attemptState.getAppAttemptTokens().getAllTokens()); + savedTokens.addAll(attemptState.getAppAttemptCredentials().getAllTokens()); assertEquals(attemptTokenSet2, savedTokens); + // attempt2 client token master key is loaded correctly + assertArrayEquals(clientTokenKey2.getEncoded(), + attemptState.getAppAttemptCredentials() + .getSecretKey(RMStateStore.AM_CLIENT_TOKEN_MASTER_KEY_NAME)); // assert store is in expected state after everything is cleaned assertTrue(stateStoreHelper.isFinalStateValid()); @@ -357,24 +365,14 @@ public class TestRMStateStore { Assert.assertEquals(sequenceNumber, secretManagerState.getDTSequenceNumber()); } - private List> generateTokens(ApplicationAttemptId attemptId, - AMRMTokenSecretManager appTokenMgr, - ClientToAMTokenSecretManagerInRM clientToAMTokenMgr, Configuration conf) { + private Token generateAMRMToken( + ApplicationAttemptId attemptId, + AMRMTokenSecretManager appTokenMgr) { AMRMTokenIdentifier appTokenId = new AMRMTokenIdentifier(attemptId); Token appToken = new Token(appTokenId, appTokenMgr); appToken.setService(new Text("appToken service")); - - ClientToAMTokenIdentifier clientToAMTokenId = - new ClientToAMTokenIdentifier(attemptId); - clientToAMTokenMgr.registerApplication(attemptId); - Token clientToAMToken = - new Token(clientToAMTokenId, clientToAMTokenMgr); - clientToAMToken.setService(new Text("clientToAMToken service")); - List> tokenPair = new ArrayList>(); - tokenPair.add(0, appToken); - tokenPair.add(1, clientToAMToken); - return tokenPair; + return appToken; } } Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java?rev=1519885&r1=1519884&r2=1519885&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java Tue Sep 3 23:20:45 2013 @@ -143,7 +143,8 @@ public class MockRMApp implements RMApp } @Override - public ApplicationReport createAndGetApplicationReport(boolean allowAccess) { + public ApplicationReport createAndGetApplicationReport( + String clientUserName, boolean allowAccess) { throw new UnsupportedOperationException("Not supported yet."); } Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java?rev=1519885&r1=1519884&r2=1519885&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java Tue Sep 3 23:20:45 2013 @@ -726,7 +726,9 @@ public class TestRMAppTransitions { public void testGetAppReport() { RMApp app = createNewTestApp(null); assertAppState(RMAppState.NEW, app); - ApplicationReport report = app.createAndGetApplicationReport(true); + ApplicationReport report = app.createAndGetApplicationReport(null, true); + Assert.assertNotNull(report.getApplicationResourceUsageReport()); + report = app.createAndGetApplicationReport("clientuser", true); Assert.assertNotNull(report.getApplicationResourceUsageReport()); } } Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java?rev=1519885&r1=1519884&r2=1519885&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java Tue Sep 3 23:20:45 2013 @@ -115,7 +115,6 @@ public class TestClientToAMTokens { private final byte[] secretKey; private InetSocketAddress address; private boolean pinged = false; - private ClientToAMTokenSecretManager secretManager; public CustomAM(ApplicationAttemptId appId, byte[] secretKey) { super("CustomAM"); @@ -132,12 +131,14 @@ public class TestClientToAMTokens { protected void serviceStart() throws Exception { Configuration conf = getConfig(); - secretManager = new ClientToAMTokenSecretManager(this.appAttemptId, secretKey); Server server; try { server = - new RPC.Builder(conf).setProtocol(CustomProtocol.class) - .setNumHandlers(1).setSecretManager(secretManager) + new RPC.Builder(conf) + .setProtocol(CustomProtocol.class) + .setNumHandlers(1) + .setSecretManager( + new ClientToAMTokenSecretManager(this.appAttemptId, secretKey)) .setInstance(this).build(); } catch (Exception e) { throw new YarnRuntimeException(e); @@ -146,14 +147,10 @@ public class TestClientToAMTokens { this.address = NetUtils.getConnectAddress(server); super.serviceStart(); } - - public ClientToAMTokenSecretManager getClientToAMTokenSecretManager() { - return this.secretManager; - } } @Test - public void testClientToAMs() throws Exception { + public void testClientToAMTokens() throws Exception { final Configuration conf = new Configuration(); conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, @@ -204,7 +201,7 @@ public class TestClientToAMTokens { GetApplicationReportResponse reportResponse = rm.getClientRMService().getApplicationReport(request); ApplicationReport appReport = reportResponse.getApplicationReport(); - org.apache.hadoop.yarn.api.records.Token clientToAMToken = + org.apache.hadoop.yarn.api.records.Token originalClientToAMToken = appReport.getClientToAMToken(); ApplicationAttemptId appAttempt = app.getCurrentAppAttempt().getAppAttemptId(); @@ -259,17 +256,47 @@ public class TestClientToAMTokens { Assert.assertFalse(am.pinged); } - // Verify denial for a malicious user - UserGroupInformation ugi = UserGroupInformation.createRemoteUser("me"); Token token = - ConverterUtils.convertFromYarn(clientToAMToken, am.address); + ConverterUtils.convertFromYarn(originalClientToAMToken, am.address); + + // Verify denial for a malicious user with tampered ID + verifyTokenWithTamperedID(conf, am, token); + + // Verify denial for a malicious user with tampered user-name + verifyTokenWithTamperedUserName(conf, am, token); + // Now for an authenticated user + verifyValidToken(conf, am, token); + } + + private void verifyTokenWithTamperedID(final Configuration conf, + final CustomAM am, Token token) + throws IOException { // Malicious user, messes with appId + UserGroupInformation ugi = UserGroupInformation.createRemoteUser("me"); ClientToAMTokenIdentifier maliciousID = new ClientToAMTokenIdentifier(BuilderUtils.newApplicationAttemptId( - BuilderUtils.newApplicationId(app.getApplicationId() - .getClusterTimestamp(), 42), 43)); + BuilderUtils.newApplicationId(am.appAttemptId.getApplicationId() + .getClusterTimestamp(), 42), 43), UserGroupInformation + .getCurrentUser().getShortUserName()); + + verifyTamperedToken(conf, am, token, ugi, maliciousID); + } + + private void verifyTokenWithTamperedUserName(final Configuration conf, + final CustomAM am, Token token) + throws IOException { + // Malicious user, messes with appId + UserGroupInformation ugi = UserGroupInformation.createRemoteUser("me"); + ClientToAMTokenIdentifier maliciousID = + new ClientToAMTokenIdentifier(am.appAttemptId, "evilOrc"); + verifyTamperedToken(conf, am, token, ugi, maliciousID); + } + + private void verifyTamperedToken(final Configuration conf, final CustomAM am, + Token token, UserGroupInformation ugi, + ClientToAMTokenIdentifier maliciousID) { Token maliciousToken = new Token(maliciousID.getBytes(), token.getPassword(), token.getKind(), @@ -309,8 +336,12 @@ public class TestClientToAMTokens { + "Mismatched response.")); Assert.assertFalse(am.pinged); } + } - // Now for an authenticated user + private void verifyValidToken(final Configuration conf, final CustomAM am, + Token token) throws IOException, + InterruptedException { + UserGroupInformation ugi; ugi = UserGroupInformation.createRemoteUser("me"); ugi.addToken(token); @@ -326,5 +357,4 @@ public class TestClientToAMTokens { } }); } - }