Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 95736117FB for ; Thu, 7 Aug 2014 07:39:06 +0000 (UTC) Received: (qmail 83354 invoked by uid 500); 7 Aug 2014 07:39:06 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 83288 invoked by uid 500); 7 Aug 2014 07:39:06 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 83279 invoked by uid 99); 7 Aug 2014 07:39:06 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 07 Aug 2014 07:39:06 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 07 Aug 2014 07:38:55 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 0F1D323895C4; Thu, 7 Aug 2014 07:38:29 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1616428 [1/2] - in /hadoop/common/branches/HDFS-6584/hadoop-common-project: hadoop-auth/ hadoop-auth/dev-support/ hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/ hadoop-auth/src/main/java/org/apache/hadoop/secur... Date: Thu, 07 Aug 2014 07:38:27 -0000 To: common-commits@hadoop.apache.org From: szetszwo@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20140807073829.0F1D323895C4@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: szetszwo Date: Thu Aug 7 07:38:23 2014 New Revision: 1616428 URL: http://svn.apache.org/r1616428 Log: Merge r1609845 through r1616427 from trunk. Added: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-auth/dev-support/ - copied from r1616427, hadoop/common/trunk/hadoop-common-project/hadoop-auth/dev-support/ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/RandomSignerSecretProvider.java - copied unchanged from r1616427, hadoop/common/trunk/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/RandomSignerSecretProvider.java hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/RolloverSignerSecretProvider.java - copied unchanged from r1616427, hadoop/common/trunk/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/RolloverSignerSecretProvider.java hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/SignerSecretProvider.java - copied unchanged from r1616427, hadoop/common/trunk/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/SignerSecretProvider.java hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/StringSignerSecretProvider.java - copied unchanged from r1616427, hadoop/common/trunk/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/StringSignerSecretProvider.java hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestRandomSignerSecretProvider.java - copied unchanged from r1616427, hadoop/common/trunk/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestRandomSignerSecretProvider.java hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestRolloverSignerSecretProvider.java - copied unchanged from r1616427, hadoop/common/trunk/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestRolloverSignerSecretProvider.java hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestStringSignerSecretProvider.java - copied unchanged from r1616427, hadoop/common/trunk/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestStringSignerSecretProvider.java hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Classpath.java - copied unchanged from r1616427, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Classpath.java hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestClasspath.java - copied unchanged from r1616427, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestClasspath.java hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJMXServlet.java - copied unchanged from r1616427, hadoop/common/trunk/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJMXServlet.java Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-auth/pom.xml hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/Signer.java hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestSigner.java hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/CHANGES.txt (contents, props changed) hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/bin/hadoop hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/ (props changed) hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Delete.java hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialShell.java hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredShell.java hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/KeyStoreTestUtil.java hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestSSLFactory.java hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/webapp/WEB-INF/web.xml hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-auth/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-auth/pom.xml?rev=1616428&r1=1616427&r2=1616428&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-auth/pom.xml (original) +++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-auth/pom.xml Thu Aug 7 07:38:23 2014 @@ -150,6 +150,13 @@ + + org.codehaus.mojo + findbugs-maven-plugin + + ${basedir}/dev-support/findbugsExcludeFile.xml + + Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java?rev=1616428&r1=1616427&r2=1616428&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java Thu Aug 7 07:38:23 2014 @@ -19,6 +19,9 @@ import org.apache.hadoop.security.authen import org.apache.hadoop.security.authentication.client.AuthenticationException; import org.apache.hadoop.security.authentication.util.Signer; import org.apache.hadoop.security.authentication.util.SignerException; +import org.apache.hadoop.security.authentication.util.RandomSignerSecretProvider; +import org.apache.hadoop.security.authentication.util.SignerSecretProvider; +import org.apache.hadoop.security.authentication.util.StringSignerSecretProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -107,11 +110,28 @@ public class AuthenticationFilter implem */ public static final String COOKIE_PATH = "cookie.path"; - private static final Random RAN = new Random(); + /** + * Constant for the configuration property that indicates the name of the + * SignerSecretProvider class to use. If not specified, SIGNATURE_SECRET + * will be used or a random secret. + */ + public static final String SIGNER_SECRET_PROVIDER_CLASS = + "signer.secret.provider"; + + /** + * Constant for the attribute that can be used for providing a custom + * object that subclasses the SignerSecretProvider. Note that this should be + * set in the ServletContext and the class should already be initialized. + * If not specified, SIGNER_SECRET_PROVIDER_CLASS will be used. + */ + public static final String SIGNATURE_PROVIDER_ATTRIBUTE = + "org.apache.hadoop.security.authentication.util.SignerSecretProvider"; private Signer signer; + private SignerSecretProvider secretProvider; private AuthenticationHandler authHandler; private boolean randomSecret; + private boolean customSecretProvider; private long validity; private String cookieDomain; private String cookiePath; @@ -159,14 +179,46 @@ public class AuthenticationFilter implem } catch (IllegalAccessException ex) { throw new ServletException(ex); } - String signatureSecret = config.getProperty(configPrefix + SIGNATURE_SECRET); - if (signatureSecret == null) { - signatureSecret = Long.toString(RAN.nextLong()); - randomSecret = true; - LOG.warn("'signature.secret' configuration not set, using a random value as secret"); + + validity = Long.parseLong(config.getProperty(AUTH_TOKEN_VALIDITY, "36000")) + * 1000; //10 hours + secretProvider = (SignerSecretProvider) filterConfig.getServletContext(). + getAttribute(SIGNATURE_PROVIDER_ATTRIBUTE); + if (secretProvider == null) { + String signerSecretProviderClassName = + config.getProperty(configPrefix + SIGNER_SECRET_PROVIDER_CLASS, null); + if (signerSecretProviderClassName == null) { + String signatureSecret = + config.getProperty(configPrefix + SIGNATURE_SECRET, null); + if (signatureSecret != null) { + secretProvider = new StringSignerSecretProvider(signatureSecret); + } else { + secretProvider = new RandomSignerSecretProvider(); + randomSecret = true; + } + } else { + try { + Class klass = Thread.currentThread().getContextClassLoader(). + loadClass(signerSecretProviderClassName); + secretProvider = (SignerSecretProvider) klass.newInstance(); + customSecretProvider = true; + } catch (ClassNotFoundException ex) { + throw new ServletException(ex); + } catch (InstantiationException ex) { + throw new ServletException(ex); + } catch (IllegalAccessException ex) { + throw new ServletException(ex); + } + } + try { + secretProvider.init(config, validity); + } catch (Exception ex) { + throw new ServletException(ex); + } + } else { + customSecretProvider = true; } - signer = new Signer(signatureSecret.getBytes()); - validity = Long.parseLong(config.getProperty(AUTH_TOKEN_VALIDITY, "36000")) * 1000; //10 hours + signer = new Signer(secretProvider); cookieDomain = config.getProperty(COOKIE_DOMAIN, null); cookiePath = config.getProperty(COOKIE_PATH, null); @@ -191,6 +243,15 @@ public class AuthenticationFilter implem } /** + * Returns if a custom implementation of a SignerSecretProvider is being used. + * + * @return if a custom implementation of a SignerSecretProvider is being used. + */ + protected boolean isCustomSignerSecretProvider() { + return customSecretProvider; + } + + /** * Returns the validity time of the generated tokens. * * @return the validity time of the generated tokens, in seconds. @@ -228,6 +289,9 @@ public class AuthenticationFilter implem authHandler.destroy(); authHandler = null; } + if (secretProvider != null) { + secretProvider.destroy(); + } } /** Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/Signer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/Signer.java?rev=1616428&r1=1616427&r2=1616428&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/Signer.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/Signer.java Thu Aug 7 07:38:23 2014 @@ -24,18 +24,19 @@ import java.security.NoSuchAlgorithmExce public class Signer { private static final String SIGNATURE = "&s="; - private byte[] secret; + private SignerSecretProvider secretProvider; /** - * Creates a Signer instance using the specified secret. + * Creates a Signer instance using the specified SignerSecretProvider. The + * SignerSecretProvider should already be initialized. * - * @param secret secret to use for creating the digest. + * @param secretProvider The SignerSecretProvider to use */ - public Signer(byte[] secret) { - if (secret == null) { - throw new IllegalArgumentException("secret cannot be NULL"); + public Signer(SignerSecretProvider secretProvider) { + if (secretProvider == null) { + throw new IllegalArgumentException("secretProvider cannot be NULL"); } - this.secret = secret.clone(); + this.secretProvider = secretProvider; } /** @@ -47,11 +48,12 @@ public class Signer { * * @return the signed string. */ - public String sign(String str) { + public synchronized String sign(String str) { if (str == null || str.length() == 0) { throw new IllegalArgumentException("NULL or empty string to sign"); } - String signature = computeSignature(str); + byte[] secret = secretProvider.getCurrentSecret(); + String signature = computeSignature(secret, str); return str + SIGNATURE + signature; } @@ -71,21 +73,19 @@ public class Signer { } String originalSignature = signedStr.substring(index + SIGNATURE.length()); String rawValue = signedStr.substring(0, index); - String currentSignature = computeSignature(rawValue); - if (!originalSignature.equals(currentSignature)) { - throw new SignerException("Invalid signature"); - } + checkSignatures(rawValue, originalSignature); return rawValue; } /** * Returns then signature of a string. * + * @param secret The secret to use * @param str string to sign. * * @return the signature for the string. */ - protected String computeSignature(String str) { + protected String computeSignature(byte[] secret, String str) { try { MessageDigest md = MessageDigest.getInstance("SHA"); md.update(str.getBytes()); @@ -97,4 +97,22 @@ public class Signer { } } + protected void checkSignatures(String rawValue, String originalSignature) + throws SignerException { + boolean isValid = false; + byte[][] secrets = secretProvider.getAllSecrets(); + for (int i = 0; i < secrets.length; i++) { + byte[] secret = secrets[i]; + if (secret != null) { + String currentSignature = computeSignature(secret, rawValue); + if (originalSignature.equals(currentSignature)) { + isValid = true; + break; + } + } + } + if (!isValid) { + throw new SignerException("Invalid signature"); + } + } } Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java?rev=1616428&r1=1616427&r2=1616428&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java Thu Aug 7 07:38:23 2014 @@ -23,6 +23,7 @@ import java.util.Vector; import javax.servlet.FilterChain; import javax.servlet.FilterConfig; +import javax.servlet.ServletContext; import javax.servlet.ServletException; import javax.servlet.ServletRequest; import javax.servlet.ServletResponse; @@ -33,6 +34,8 @@ import javax.servlet.http.HttpServletRes import org.apache.hadoop.security.authentication.client.AuthenticatedURL; import org.apache.hadoop.security.authentication.client.AuthenticationException; import org.apache.hadoop.security.authentication.util.Signer; +import org.apache.hadoop.security.authentication.util.SignerSecretProvider; +import org.apache.hadoop.security.authentication.util.StringSignerSecretProvider; import org.junit.Assert; import org.junit.Test; import org.mockito.Mockito; @@ -157,9 +160,14 @@ public class TestAuthenticationFilter { Mockito.when(config.getInitParameterNames()).thenReturn( new Vector(Arrays.asList(AuthenticationFilter.AUTH_TYPE, AuthenticationFilter.AUTH_TOKEN_VALIDITY)).elements()); + ServletContext context = Mockito.mock(ServletContext.class); + Mockito.when(context.getAttribute( + AuthenticationFilter.SIGNATURE_PROVIDER_ATTRIBUTE)).thenReturn(null); + Mockito.when(config.getServletContext()).thenReturn(context); filter.init(config); Assert.assertEquals(PseudoAuthenticationHandler.class, filter.getAuthenticationHandler().getClass()); Assert.assertTrue(filter.isRandomSecret()); + Assert.assertFalse(filter.isCustomSignerSecretProvider()); Assert.assertNull(filter.getCookieDomain()); Assert.assertNull(filter.getCookiePath()); Assert.assertEquals(TOKEN_VALIDITY_SEC, filter.getValidity()); @@ -167,6 +175,26 @@ public class TestAuthenticationFilter { filter.destroy(); } + // string secret + filter = new AuthenticationFilter(); + try { + FilterConfig config = Mockito.mock(FilterConfig.class); + Mockito.when(config.getInitParameter(AuthenticationFilter.AUTH_TYPE)).thenReturn("simple"); + Mockito.when(config.getInitParameter(AuthenticationFilter.SIGNATURE_SECRET)).thenReturn("secret"); + Mockito.when(config.getInitParameterNames()).thenReturn( + new Vector(Arrays.asList(AuthenticationFilter.AUTH_TYPE, + AuthenticationFilter.SIGNATURE_SECRET)).elements()); + ServletContext context = Mockito.mock(ServletContext.class); + Mockito.when(context.getAttribute( + AuthenticationFilter.SIGNATURE_PROVIDER_ATTRIBUTE)).thenReturn(null); + Mockito.when(config.getServletContext()).thenReturn(context); + filter.init(config); + Assert.assertFalse(filter.isRandomSecret()); + Assert.assertFalse(filter.isCustomSignerSecretProvider()); + } finally { + filter.destroy(); + } + // custom secret filter = new AuthenticationFilter(); try { @@ -176,8 +204,26 @@ public class TestAuthenticationFilter { Mockito.when(config.getInitParameterNames()).thenReturn( new Vector(Arrays.asList(AuthenticationFilter.AUTH_TYPE, AuthenticationFilter.SIGNATURE_SECRET)).elements()); + ServletContext context = Mockito.mock(ServletContext.class); + Mockito.when(context.getAttribute( + AuthenticationFilter.SIGNATURE_PROVIDER_ATTRIBUTE)).thenReturn( + new SignerSecretProvider() { + @Override + public void init(Properties config, long tokenValidity) { + } + @Override + public byte[] getCurrentSecret() { + return null; + } + @Override + public byte[][] getAllSecrets() { + return null; + } + }); + Mockito.when(config.getServletContext()).thenReturn(context); filter.init(config); Assert.assertFalse(filter.isRandomSecret()); + Assert.assertTrue(filter.isCustomSignerSecretProvider()); } finally { filter.destroy(); } @@ -193,6 +239,10 @@ public class TestAuthenticationFilter { new Vector(Arrays.asList(AuthenticationFilter.AUTH_TYPE, AuthenticationFilter.COOKIE_DOMAIN, AuthenticationFilter.COOKIE_PATH)).elements()); + ServletContext context = Mockito.mock(ServletContext.class); + Mockito.when(context.getAttribute( + AuthenticationFilter.SIGNATURE_PROVIDER_ATTRIBUTE)).thenReturn(null); + Mockito.when(config.getServletContext()).thenReturn(context); filter.init(config); Assert.assertEquals(".foo.com", filter.getCookieDomain()); Assert.assertEquals("/bar", filter.getCookiePath()); @@ -213,6 +263,10 @@ public class TestAuthenticationFilter { new Vector( Arrays.asList(AuthenticationFilter.AUTH_TYPE, "management.operation.return")).elements()); + ServletContext context = Mockito.mock(ServletContext.class); + Mockito.when(context.getAttribute( + AuthenticationFilter.SIGNATURE_PROVIDER_ATTRIBUTE)).thenReturn(null); + Mockito.when(config.getServletContext()).thenReturn(context); filter.init(config); Assert.assertTrue(DummyAuthenticationHandler.init); } finally { @@ -248,6 +302,10 @@ public class TestAuthenticationFilter { Mockito.when(config.getInitParameterNames()).thenReturn( new Vector(Arrays.asList(AuthenticationFilter.AUTH_TYPE, AuthenticationFilter.AUTH_TOKEN_VALIDITY)).elements()); + ServletContext context = Mockito.mock(ServletContext.class); + Mockito.when(context.getAttribute( + AuthenticationFilter.SIGNATURE_PROVIDER_ATTRIBUTE)).thenReturn(null); + Mockito.when(config.getServletContext()).thenReturn(context); filter.init(config); Assert.assertEquals(PseudoAuthenticationHandler.class, @@ -270,6 +328,10 @@ public class TestAuthenticationFilter { new Vector( Arrays.asList(AuthenticationFilter.AUTH_TYPE, "management.operation.return")).elements()); + ServletContext context = Mockito.mock(ServletContext.class); + Mockito.when(context.getAttribute( + AuthenticationFilter.SIGNATURE_PROVIDER_ATTRIBUTE)).thenReturn(null); + Mockito.when(config.getServletContext()).thenReturn(context); filter.init(config); HttpServletRequest request = Mockito.mock(HttpServletRequest.class); @@ -297,11 +359,15 @@ public class TestAuthenticationFilter { Arrays.asList(AuthenticationFilter.AUTH_TYPE, AuthenticationFilter.SIGNATURE_SECRET, "management.operation.return")).elements()); + ServletContext context = Mockito.mock(ServletContext.class); + Mockito.when(context.getAttribute( + AuthenticationFilter.SIGNATURE_PROVIDER_ATTRIBUTE)).thenReturn(null); + Mockito.when(config.getServletContext()).thenReturn(context); filter.init(config); AuthenticationToken token = new AuthenticationToken("u", "p", DummyAuthenticationHandler.TYPE); token.setExpires(System.currentTimeMillis() + TOKEN_VALIDITY_SEC); - Signer signer = new Signer("secret".getBytes()); + Signer signer = new Signer(new StringSignerSecretProvider("secret")); String tokenSigned = signer.sign(token.toString()); Cookie cookie = new Cookie(AuthenticatedURL.AUTH_COOKIE, tokenSigned); @@ -330,12 +396,16 @@ public class TestAuthenticationFilter { Arrays.asList(AuthenticationFilter.AUTH_TYPE, AuthenticationFilter.SIGNATURE_SECRET, "management.operation.return")).elements()); + ServletContext context = Mockito.mock(ServletContext.class); + Mockito.when(context.getAttribute( + AuthenticationFilter.SIGNATURE_PROVIDER_ATTRIBUTE)).thenReturn(null); + Mockito.when(config.getServletContext()).thenReturn(context); filter.init(config); AuthenticationToken token = new AuthenticationToken("u", "p", DummyAuthenticationHandler.TYPE); token.setExpires(System.currentTimeMillis() - TOKEN_VALIDITY_SEC); - Signer signer = new Signer("secret".getBytes()); + Signer signer = new Signer(new StringSignerSecretProvider("secret")); String tokenSigned = signer.sign(token.toString()); Cookie cookie = new Cookie(AuthenticatedURL.AUTH_COOKIE, tokenSigned); @@ -371,11 +441,15 @@ public class TestAuthenticationFilter { Arrays.asList(AuthenticationFilter.AUTH_TYPE, AuthenticationFilter.SIGNATURE_SECRET, "management.operation.return")).elements()); + ServletContext context = Mockito.mock(ServletContext.class); + Mockito.when(context.getAttribute( + AuthenticationFilter.SIGNATURE_PROVIDER_ATTRIBUTE)).thenReturn(null); + Mockito.when(config.getServletContext()).thenReturn(context); filter.init(config); AuthenticationToken token = new AuthenticationToken("u", "p", "invalidtype"); token.setExpires(System.currentTimeMillis() + TOKEN_VALIDITY_SEC); - Signer signer = new Signer("secret".getBytes()); + Signer signer = new Signer(new StringSignerSecretProvider("secret")); String tokenSigned = signer.sign(token.toString()); Cookie cookie = new Cookie(AuthenticatedURL.AUTH_COOKIE, tokenSigned); @@ -409,6 +483,10 @@ public class TestAuthenticationFilter { new Vector( Arrays.asList(AuthenticationFilter.AUTH_TYPE, "management.operation.return")).elements()); + ServletContext context = Mockito.mock(ServletContext.class); + Mockito.when(context.getAttribute( + AuthenticationFilter.SIGNATURE_PROVIDER_ATTRIBUTE)).thenReturn(null); + Mockito.when(config.getServletContext()).thenReturn(context); filter.init(config); HttpServletRequest request = Mockito.mock(HttpServletRequest.class); @@ -458,6 +536,10 @@ public class TestAuthenticationFilter { AuthenticationFilter.AUTH_TOKEN_VALIDITY, AuthenticationFilter.SIGNATURE_SECRET, "management.operation" + ".return", "expired.token")).elements()); + ServletContext context = Mockito.mock(ServletContext.class); + Mockito.when(context.getAttribute( + AuthenticationFilter.SIGNATURE_PROVIDER_ATTRIBUTE)).thenReturn(null); + Mockito.when(config.getServletContext()).thenReturn(context); if (withDomainPath) { Mockito.when(config.getInitParameter(AuthenticationFilter @@ -511,7 +593,7 @@ public class TestAuthenticationFilter { Mockito.verify(chain).doFilter(Mockito.any(ServletRequest.class), Mockito.any(ServletResponse.class)); - Signer signer = new Signer("secret".getBytes()); + Signer signer = new Signer(new StringSignerSecretProvider("secret")); String value = signer.verifyAndExtract(v); AuthenticationToken token = AuthenticationToken.parse(value); assertThat(token.getExpires(), not(0L)); @@ -578,6 +660,10 @@ public class TestAuthenticationFilter { new Vector( Arrays.asList(AuthenticationFilter.AUTH_TYPE, "management.operation.return")).elements()); + ServletContext context = Mockito.mock(ServletContext.class); + Mockito.when(context.getAttribute( + AuthenticationFilter.SIGNATURE_PROVIDER_ATTRIBUTE)).thenReturn(null); + Mockito.when(config.getServletContext()).thenReturn(context); filter.init(config); HttpServletRequest request = Mockito.mock(HttpServletRequest.class); @@ -585,7 +671,7 @@ public class TestAuthenticationFilter { AuthenticationToken token = new AuthenticationToken("u", "p", "t"); token.setExpires(System.currentTimeMillis() + TOKEN_VALIDITY_SEC); - Signer signer = new Signer("secret".getBytes()); + Signer signer = new Signer(new StringSignerSecretProvider("secret")); String tokenSigned = signer.sign(token.toString()); Cookie cookie = new Cookie(AuthenticatedURL.AUTH_COOKIE, tokenSigned); @@ -628,6 +714,10 @@ public class TestAuthenticationFilter { new Vector( Arrays.asList(AuthenticationFilter.AUTH_TYPE, "management.operation.return")).elements()); + ServletContext context = Mockito.mock(ServletContext.class); + Mockito.when(context.getAttribute( + AuthenticationFilter.SIGNATURE_PROVIDER_ATTRIBUTE)).thenReturn(null); + Mockito.when(config.getServletContext()).thenReturn(context); filter.init(config); HttpServletRequest request = Mockito.mock(HttpServletRequest.class); @@ -691,6 +781,10 @@ public class TestAuthenticationFilter { Arrays.asList(AuthenticationFilter.AUTH_TYPE, AuthenticationFilter.SIGNATURE_SECRET, "management.operation.return")).elements()); + ServletContext context = Mockito.mock(ServletContext.class); + Mockito.when(context.getAttribute( + AuthenticationFilter.SIGNATURE_PROVIDER_ATTRIBUTE)).thenReturn(null); + Mockito.when(config.getServletContext()).thenReturn(context); filter.init(config); HttpServletRequest request = Mockito.mock(HttpServletRequest.class); @@ -698,7 +792,7 @@ public class TestAuthenticationFilter { AuthenticationToken token = new AuthenticationToken("u", "p", DummyAuthenticationHandler.TYPE); token.setExpires(System.currentTimeMillis() - TOKEN_VALIDITY_SEC); - Signer signer = new Signer(secret.getBytes()); + Signer signer = new Signer(new StringSignerSecretProvider(secret)); String tokenSigned = signer.sign(token.toString()); Cookie cookie = new Cookie(AuthenticatedURL.AUTH_COOKIE, tokenSigned); @@ -758,6 +852,10 @@ public class TestAuthenticationFilter { Arrays.asList(AuthenticationFilter.AUTH_TYPE, AuthenticationFilter.SIGNATURE_SECRET, "management.operation.return")).elements()); + ServletContext context = Mockito.mock(ServletContext.class); + Mockito.when(context.getAttribute( + AuthenticationFilter.SIGNATURE_PROVIDER_ATTRIBUTE)).thenReturn(null); + Mockito.when(config.getServletContext()).thenReturn(context); filter.init(config); HttpServletRequest request = Mockito.mock(HttpServletRequest.class); @@ -765,7 +863,7 @@ public class TestAuthenticationFilter { AuthenticationToken token = new AuthenticationToken("u", "p", "invalidtype"); token.setExpires(System.currentTimeMillis() + TOKEN_VALIDITY_SEC); - Signer signer = new Signer(secret.getBytes()); + Signer signer = new Signer(new StringSignerSecretProvider(secret)); String tokenSigned = signer.sign(token.toString()); Cookie cookie = new Cookie(AuthenticatedURL.AUTH_COOKIE, tokenSigned); @@ -793,6 +891,10 @@ public class TestAuthenticationFilter { new Vector( Arrays.asList(AuthenticationFilter.AUTH_TYPE, "management.operation.return")).elements()); + ServletContext context = Mockito.mock(ServletContext.class); + Mockito.when(context.getAttribute( + AuthenticationFilter.SIGNATURE_PROVIDER_ATTRIBUTE)).thenReturn(null); + Mockito.when(config.getServletContext()).thenReturn(context); filter.init(config); HttpServletRequest request = Mockito.mock(HttpServletRequest.class); @@ -812,7 +914,7 @@ public class TestAuthenticationFilter { AuthenticationToken token = new AuthenticationToken("u", "p", "t"); token.setExpires(System.currentTimeMillis() + TOKEN_VALIDITY_SEC); - Signer signer = new Signer("secret".getBytes()); + Signer signer = new Signer(new StringSignerSecretProvider("secret")); String tokenSigned = signer.sign(token.toString()); Cookie cookie = new Cookie(AuthenticatedURL.AUTH_COOKIE, tokenSigned); Mockito.when(request.getCookies()).thenReturn(new Cookie[]{cookie}); Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestSigner.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestSigner.java?rev=1616428&r1=1616427&r2=1616428&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestSigner.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestSigner.java Thu Aug 7 07:38:23 2014 @@ -13,24 +13,15 @@ */ package org.apache.hadoop.security.authentication.util; +import java.util.Properties; import org.junit.Assert; import org.junit.Test; public class TestSigner { @Test - public void testNoSecret() throws Exception { - try { - new Signer(null); - Assert.fail(); - } - catch (IllegalArgumentException ex) { - } - } - - @Test public void testNullAndEmptyString() throws Exception { - Signer signer = new Signer("secret".getBytes()); + Signer signer = new Signer(new StringSignerSecretProvider("secret")); try { signer.sign(null); Assert.fail(); @@ -51,17 +42,17 @@ public class TestSigner { @Test public void testSignature() throws Exception { - Signer signer = new Signer("secret".getBytes()); + Signer signer = new Signer(new StringSignerSecretProvider("secret")); String s1 = signer.sign("ok"); String s2 = signer.sign("ok"); String s3 = signer.sign("wrong"); Assert.assertEquals(s1, s2); - Assert.assertNotSame(s1, s3); + Assert.assertNotEquals(s1, s3); } @Test public void testVerify() throws Exception { - Signer signer = new Signer("secret".getBytes()); + Signer signer = new Signer(new StringSignerSecretProvider("secret")); String t = "test"; String s = signer.sign(t); String e = signer.verifyAndExtract(s); @@ -70,7 +61,7 @@ public class TestSigner { @Test public void testInvalidSignedText() throws Exception { - Signer signer = new Signer("secret".getBytes()); + Signer signer = new Signer(new StringSignerSecretProvider("secret")); try { signer.verifyAndExtract("test"); Assert.fail(); @@ -83,7 +74,7 @@ public class TestSigner { @Test public void testTampering() throws Exception { - Signer signer = new Signer("secret".getBytes()); + Signer signer = new Signer(new StringSignerSecretProvider("secret")); String t = "test"; String s = signer.sign(t); s += "x"; @@ -96,4 +87,66 @@ public class TestSigner { Assert.fail(); } } + + @Test + public void testMultipleSecrets() throws Exception { + TestSignerSecretProvider secretProvider = new TestSignerSecretProvider(); + Signer signer = new Signer(secretProvider); + secretProvider.setCurrentSecret("secretB"); + String t1 = "test"; + String s1 = signer.sign(t1); + String e1 = signer.verifyAndExtract(s1); + Assert.assertEquals(t1, e1); + secretProvider.setPreviousSecret("secretA"); + String t2 = "test"; + String s2 = signer.sign(t2); + String e2 = signer.verifyAndExtract(s2); + Assert.assertEquals(t2, e2); + Assert.assertEquals(s1, s2); //check is using current secret for signing + secretProvider.setCurrentSecret("secretC"); + secretProvider.setPreviousSecret("secretB"); + String t3 = "test"; + String s3 = signer.sign(t3); + String e3 = signer.verifyAndExtract(s3); + Assert.assertEquals(t3, e3); + Assert.assertNotEquals(s1, s3); //check not using current secret for signing + String e1b = signer.verifyAndExtract(s1); + Assert.assertEquals(t1, e1b); // previous secret still valid + secretProvider.setCurrentSecret("secretD"); + secretProvider.setPreviousSecret("secretC"); + try { + signer.verifyAndExtract(s1); // previous secret no longer valid + Assert.fail(); + } catch (SignerException ex) { + // Expected + } + } + + class TestSignerSecretProvider extends SignerSecretProvider { + + private byte[] currentSecret; + private byte[] previousSecret; + + @Override + public void init(Properties config, long tokenValidity) { + } + + @Override + public byte[] getCurrentSecret() { + return currentSecret; + } + + @Override + public byte[][] getAllSecrets() { + return new byte[][]{currentSecret, previousSecret}; + } + + public void setCurrentSecret(String secretStr) { + currentSecret = secretStr.getBytes(); + } + + public void setPreviousSecret(String previousSecretStr) { + previousSecret = previousSecretStr.getBytes(); + } + } } Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1616428&r1=1616427&r2=1616428&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/CHANGES.txt Thu Aug 7 07:38:23 2014 @@ -195,6 +195,10 @@ Trunk (Unreleased) HADOOP-10756. KMS audit log should consolidate successful similar requests. (asuresh via tucu) + HADOOP-10793. KeyShell args should use single-dash style. (wang) + + HADOOP-10936. Change default KeyProvider bitlength to 128. (wang) + BUG FIXES HADOOP-9451. Fault single-layer config if node group topology is enabled. @@ -408,6 +412,15 @@ Trunk (Unreleased) HADOOP-10881. Clarify usage of encryption and encrypted encryption key in KeyProviderCryptoExtension. (wang) + HADOOP-10920. site plugin couldn't parse hadoop-kms index.apt.vm. + (Akira Ajisaka via wang) + + HADOOP-10925. Compilation fails in native link0 function on Windows. + (cnauroth) + + HADOOP-10939. Fix TestKeyProviderFactory testcases to use default 128 bit + length keys. (Arun Suresh via wang) + OPTIMIZATIONS HADOOP-7761. Improve the performance of raw comparisons. (todd) @@ -466,6 +479,17 @@ Release 2.6.0 - UNRELEASED HADOOP-8069. Enable TCP_NODELAY by default for IPC. (Todd Lipcon via Arpit Agarwal) + HADOOP-10902. Deletion of directories with snapshots will not output + reason for trash move failure. (Stephen Chu via wang) + + HADOOP-10900. CredentialShell args should use single-dash style. (wang) + + HADOOP-10903. Enhance hadoop classpath command to expand wildcards or write + classpath into jar manifest. (cnauroth) + + HADOOP-10791. AuthenticationFilter should support externalizing the + secret for signing and provide rotation support. (rkanter via tucu) + OPTIMIZATIONS BUG FIXES @@ -500,6 +524,25 @@ Release 2.6.0 - UNRELEASED HADOOP-10876. The constructor of Path should not take an empty URL as a parameter. (Zhihai Xu via wang) + HADOOP-10928. Incorrect usage on `hadoop credential list`. + (Josh Elser via wang) + + HADOOP-10927. Fix CredentialShell help behavior and error codes. + (Josh Elser via wang) + + HADOOP-10937. Need to set version name correctly before decrypting EEK. + (Arun Suresh via wang) + + HADOOP-10918. JMXJsonServlet fails when used within Tomcat. (tucu) + + HADOOP-10933. FileBasedKeyStoresFactory Should use Configuration.getPassword + for SSL Passwords. (lmccay via tucu) + + HADOOP-10759. Remove hardcoded JAVA_HEAP_MAX. (Sam Liu via Eric Yang) + + HADOOP-10905. LdapGroupsMapping Should use configuration.getPassword for SSL + and LDAP Passwords. (lmccay via brandonli) + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES Propchange: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/CHANGES.txt ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1615020-1616427 Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/bin/hadoop URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/bin/hadoop?rev=1616428&r1=1616427&r2=1616428&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/bin/hadoop (original) +++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/bin/hadoop Thu Aug 7 07:38:23 2014 @@ -35,6 +35,7 @@ function print_usage(){ echo " distcp copy file or directories recursively" echo " archive -archiveName NAME -p * create a hadoop archive" echo " classpath prints the class path needed to get the" + echo " credential interact with credential providers" echo " Hadoop jar and the required libraries" echo " daemonlog get/set the log level for each daemon" echo " or" @@ -90,11 +91,6 @@ case $COMMAND in fi ;; - classpath) - echo $CLASSPATH - exit - ;; - #core commands *) # the core commands @@ -118,6 +114,14 @@ case $COMMAND in CLASSPATH=${CLASSPATH}:${TOOL_PATH} elif [ "$COMMAND" = "credential" ] ; then CLASS=org.apache.hadoop.security.alias.CredentialShell + elif [ "$COMMAND" = "classpath" ] ; then + if [ "$#" -eq 1 ]; then + # No need to bother starting up a JVM for this simple case. + echo $CLASSPATH + exit + else + CLASS=org.apache.hadoop.util.Classpath + fi elif [[ "$COMMAND" = -* ]] ; then # class and package names cannot begin with a - echo "Error: No command named \`$COMMAND' was found. Perhaps you meant \`hadoop ${COMMAND#-}'" Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh?rev=1616428&r1=1616427&r2=1616428&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh (original) +++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh Thu Aug 7 07:38:23 2014 @@ -149,8 +149,6 @@ if [[ -z $JAVA_HOME ]]; then fi JAVA=$JAVA_HOME/bin/java -# some Java parameters -JAVA_HEAP_MAX=-Xmx1000m # check envvars which might override default args if [ "$HADOOP_HEAPSIZE" != "" ]; then Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd?rev=1616428&r1=1616427&r2=1616428&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd (original) +++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd Thu Aug 7 07:38:23 2014 @@ -115,11 +115,14 @@ call :updatepath %HADOOP_BIN_PATH% ) if %hadoop-command% == classpath ( - @echo %CLASSPATH% - goto :eof + if not defined hadoop-command-arguments ( + @rem No need to bother starting up a JVM for this simple case. + @echo %CLASSPATH% + exit /b + ) ) - set corecommands=fs version jar checknative distcp daemonlog archive + set corecommands=fs version jar checknative distcp daemonlog archive classpath for %%i in ( %corecommands% ) do ( if %hadoop-command% == %%i set corecommand=true ) @@ -175,6 +178,10 @@ call :updatepath %HADOOP_BIN_PATH% set CLASSPATH=%CLASSPATH%;%TOOL_PATH% goto :eof +:classpath + set CLASS=org.apache.hadoop.util.Classpath + goto :eof + :updatepath set path_to_add=%* set current_path_comparable=%path% Propchange: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1615020-1616427 Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java?rev=1616428&r1=1616427&r2=1616428&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java Thu Aug 7 07:38:23 2014 @@ -54,7 +54,7 @@ public abstract class KeyProvider { public static final String DEFAULT_CIPHER = "AES/CTR/NoPadding"; public static final String DEFAULT_BITLENGTH_NAME = "hadoop.security.key.default.bitlength"; - public static final int DEFAULT_BITLENGTH = 256; + public static final int DEFAULT_BITLENGTH = 128; /** * The combination of both the key version name and the key material. @@ -341,6 +341,16 @@ public abstract class KeyProvider { public Map getAttributes() { return (attributes == null) ? Collections.EMPTY_MAP : attributes; } + + @Override + public String toString() { + return "Options{" + + "cipher='" + cipher + '\'' + + ", bitLength=" + bitLength + + ", description='" + description + '\'' + + ", attributes=" + attributes + + '}'; + } } /** Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java?rev=1616428&r1=1616427&r2=1616428&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java Thu Aug 7 07:38:23 2014 @@ -21,11 +21,13 @@ package org.apache.hadoop.crypto.key; import java.io.IOException; import java.security.GeneralSecurityException; import java.security.SecureRandom; + import javax.crypto.Cipher; import javax.crypto.spec.IvParameterSpec; import javax.crypto.spec.SecretKeySpec; import com.google.common.base.Preconditions; + import org.apache.hadoop.classification.InterfaceAudience; /** @@ -97,7 +99,7 @@ public class KeyProviderCryptoExtension public static EncryptedKeyVersion createForDecryption(String encryptionKeyVersionName, byte[] encryptedKeyIv, byte[] encryptedKeyMaterial) { - KeyVersion encryptedKeyVersion = new KeyVersion(null, null, + KeyVersion encryptedKeyVersion = new KeyVersion(null, EEK, encryptedKeyMaterial); return new EncryptedKeyVersion(null, encryptionKeyVersionName, encryptedKeyIv, encryptedKeyVersion); @@ -258,6 +260,13 @@ public class KeyProviderCryptoExtension keyProvider.getKeyVersion(encryptionKeyVersionName); Preconditions.checkNotNull(encryptionKey, "KeyVersion name '%s' does not exist", encryptionKeyVersionName); + Preconditions.checkArgument( + encryptedKeyVersion.getEncryptedKeyVersion().getVersionName() + .equals(KeyProviderCryptoExtension.EEK), + "encryptedKey version name must be '%s', is '%s'", + KeyProviderCryptoExtension.EEK, + encryptedKeyVersion.getEncryptedKeyVersion().getVersionName() + ); final byte[] encryptionKeyMaterial = encryptionKey.getMaterial(); // Encryption key IV is determined from encrypted key's IV final byte[] encryptionIV = Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java?rev=1616428&r1=1616427&r2=1616428&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java Thu Aug 7 07:38:23 2014 @@ -38,9 +38,9 @@ import org.apache.hadoop.util.ToolRunner */ public class KeyShell extends Configured implements Tool { final static private String USAGE_PREFIX = "Usage: hadoop key " + - "[generic options]\n"; + "[generic options]\n"; final static private String COMMANDS = - " [--help]\n" + + " [-help]\n" + " [" + CreateCommand.USAGE + "]\n" + " [" + RollCommand.USAGE + "]\n" + " [" + DeleteCommand.USAGE + "]\n" + @@ -90,11 +90,11 @@ public class KeyShell extends Configured /** * Parse the command line arguments and initialize the data * 
-   * % hadoop key create keyName [--size size] [--cipher algorithm]
-   *    [--provider providerPath]
-   * % hadoop key roll keyName [--provider providerPath]
+   * % hadoop key create keyName [-size size] [-cipher algorithm]
+   *    [-provider providerPath]
+   * % hadoop key roll keyName [-provider providerPath]
    * % hadoop key list [-provider providerPath]
-   * % hadoop key delete keyName [--provider providerPath] [-i]
+   * % hadoop key delete keyName [-provider providerPath] [-i]
    *
* @param args Command line arguments. * @return 0 on success, 1 on failure. @@ -107,47 +107,47 @@ public class KeyShell extends Configured for (int i = 0; i < args.length; i++) { // parse command line boolean moreTokens = (i < args.length - 1); if (args[i].equals("create")) { - String keyName = "--help"; + String keyName = "-help"; if (moreTokens) { keyName = args[++i]; } command = new CreateCommand(keyName, options); - if ("--help".equals(keyName)) { + if ("-help".equals(keyName)) { printKeyShellUsage(); return 1; } } else if (args[i].equals("delete")) { - String keyName = "--help"; + String keyName = "-help"; if (moreTokens) { keyName = args[++i]; } command = new DeleteCommand(keyName); - if ("--help".equals(keyName)) { + if ("-help".equals(keyName)) { printKeyShellUsage(); return 1; } } else if (args[i].equals("roll")) { - String keyName = "--help"; + String keyName = "-help"; if (moreTokens) { keyName = args[++i]; } command = new RollCommand(keyName); - if ("--help".equals(keyName)) { + if ("-help".equals(keyName)) { printKeyShellUsage(); return 1; } } else if ("list".equals(args[i])) { command = new ListCommand(); - } else if ("--size".equals(args[i]) && moreTokens) { + } else if ("-size".equals(args[i]) && moreTokens) { options.setBitLength(Integer.parseInt(args[++i])); - } else if ("--cipher".equals(args[i]) && moreTokens) { + } else if ("-cipher".equals(args[i]) && moreTokens) { options.setCipher(args[++i]); - } else if ("--description".equals(args[i]) && moreTokens) { + } else if ("-description".equals(args[i]) && moreTokens) { options.setDescription(args[++i]); - } else if ("--attr".equals(args[i]) && moreTokens) { + } else if ("-attr".equals(args[i]) && moreTokens) { final String attrval[] = args[++i].split("=", 2); final String attr = attrval[0].trim(); final String val = attrval[1].trim(); @@ -164,14 +164,14 @@ public class KeyShell extends Configured return 1; } attributes.put(attr, val); - } else if ("--provider".equals(args[i]) && moreTokens) { + } else if ("-provider".equals(args[i]) && moreTokens) { userSuppliedProvider = true; getConf().set(KeyProviderFactory.KEY_PROVIDER_PATH, args[++i]); - } else if ("--metadata".equals(args[i])) { + } else if ("-metadata".equals(args[i])) { getConf().setBoolean(LIST_METADATA, true); - } else if ("-i".equals(args[i]) || ("--interactive".equals(args[i]))) { + } else if ("-i".equals(args[i]) || ("-interactive".equals(args[i]))) { interactive = true; - } else if ("--help".equals(args[i])) { + } else if ("-help".equals(args[i])) { printKeyShellUsage(); return 1; } else { @@ -258,11 +258,11 @@ public class KeyShell extends Configured private class ListCommand extends Command { public static final String USAGE = - "list [--provider ] [--metadata] [--help]"; + "list [-provider ] [-metadata] [-help]"; public static final String DESC = "The list subcommand displays the keynames contained within\n" + "a particular provider as configured in core-site.xml or\n" + - "specified with the --provider argument. --metadata displays\n" + + "specified with the -provider argument. -metadata displays\n" + "the metadata."; private boolean metadata = false; @@ -272,9 +272,9 @@ public class KeyShell extends Configured provider = getKeyProvider(); if (provider == null) { out.println("There are no non-transient KeyProviders configured.\n" - + "Use the --provider option to specify a provider. If you\n" + + "Use the -provider option to specify a provider. If you\n" + "want to list a transient provider then you must use the\n" - + "--provider argument."); + + "-provider argument."); rc = false; } metadata = getConf().getBoolean(LIST_METADATA, false); @@ -310,10 +310,10 @@ public class KeyShell extends Configured } private class RollCommand extends Command { - public static final String USAGE = "roll [--provider ] [--help]"; + public static final String USAGE = "roll [-provider ] [-help]"; public static final String DESC = "The roll subcommand creates a new version for the specified key\n" + - "within the provider indicated using the --provider argument\n"; + "within the provider indicated using the -provider argument\n"; String keyName = null; @@ -326,13 +326,13 @@ public class KeyShell extends Configured provider = getKeyProvider(); if (provider == null) { out.println("There are no valid KeyProviders configured. The key\n" + - "has not been rolled. Use the --provider option to specify\n" + + "has not been rolled. Use the -provider option to specify\n" + "a provider."); rc = false; } if (keyName == null) { out.println("Please provide a .\n" + - "See the usage description by using --help."); + "See the usage description by using -help."); rc = false; } return rc; @@ -367,11 +367,11 @@ public class KeyShell extends Configured } private class DeleteCommand extends Command { - public static final String USAGE = "delete [--provider ] [--help]"; + public static final String USAGE = "delete [-provider ] [-help]"; public static final String DESC = "The delete subcommand deletes all versions of the key\n" + "specified by the argument from within the\n" + - "provider specified --provider."; + "provider specified -provider."; String keyName = null; boolean cont = true; @@ -385,12 +385,12 @@ public class KeyShell extends Configured provider = getKeyProvider(); if (provider == null) { out.println("There are no valid KeyProviders configured. Nothing\n" - + "was deleted. Use the --provider option to specify a provider."); + + "was deleted. Use the -provider option to specify a provider."); return false; } if (keyName == null) { out.println("There is no keyName specified. Please specify a " + - ". See the usage description with --help."); + ". See the usage description with -help."); return false; } if (interactive) { @@ -436,19 +436,19 @@ public class KeyShell extends Configured private class CreateCommand extends Command { public static final String USAGE = - "create [--cipher ] [--size ]\n" + - " [--description ]\n" + - " [--attr ]\n" + - " [--provider ] [--help]"; + "create [-cipher ] [-size ]\n" + + " [-description ]\n" + + " [-attr ]\n" + + " [-provider ] [-help]"; public static final String DESC = "The create subcommand creates a new key for the name specified\n" + "by the argument within the provider specified by the\n" + - "--provider argument. You may specify a cipher with the --cipher\n" + + "-provider argument. You may specify a cipher with the -cipher\n" + "argument. The default cipher is currently \"AES/CTR/NoPadding\".\n" + - "The default keysize is 256. You may specify the requested key\n" + - "length using the --size argument. Arbitrary attribute=value\n" + - "style attributes may be specified using the --attr argument.\n" + - "--attr may be specified multiple times, once per attribute.\n"; + "The default keysize is 128. You may specify the requested key\n" + + "length using the -size argument. Arbitrary attribute=value\n" + + "style attributes may be specified using the -attr argument.\n" + + "-attr may be specified multiple times, once per attribute.\n"; final String keyName; final Options options; @@ -463,13 +463,13 @@ public class KeyShell extends Configured provider = getKeyProvider(); if (provider == null) { out.println("There are no valid KeyProviders configured. No key\n" + - " was created. You can use the --provider option to specify\n" + + " was created. You can use the -provider option to specify\n" + " a provider to use."); rc = false; } if (keyName == null) { out.println("Please provide a . See the usage description" + - " with --help."); + " with -help."); rc = false; } return rc; @@ -479,7 +479,8 @@ public class KeyShell extends Configured warnIfTransientProvider(); try { provider.createKey(keyName, options); - out.println(keyName + " has been successfully created."); + out.println(keyName + " has been successfully created with options " + + options.toString() + "."); provider.flush(); printProviderWritten(); } catch (InvalidParameterException e) { Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java?rev=1616428&r1=1616427&r2=1616428&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java Thu Aug 7 07:38:23 2014 @@ -653,7 +653,7 @@ public class KMSClientProvider extends K encryptedKeyVersion.getEncryptedKeyVersion().getVersionName() .equals(KeyProviderCryptoExtension.EEK), "encryptedKey version name must be '%s', is '%s'", - KeyProviderCryptoExtension.EK, + KeyProviderCryptoExtension.EEK, encryptedKeyVersion.getEncryptedKeyVersion().getVersionName() ); checkNotNull(encryptedKeyVersion.getEncryptedKeyVersion(), "encryptedKey"); Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Delete.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Delete.java?rev=1616428&r1=1616427&r2=1616428&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Delete.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Delete.java Thu Aug 7 07:38:23 2014 @@ -118,7 +118,11 @@ class Delete { } catch(FileNotFoundException fnfe) { throw fnfe; } catch (IOException ioe) { - throw new IOException(ioe.getMessage() + ". Consider using -skipTrash option", ioe); + String msg = ioe.getMessage(); + if (ioe.getCause() != null) { + msg += ": " + ioe.getCause().getMessage(); + } + throw new IOException(msg + ". Consider using -skipTrash option", ioe); } } return success; Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java?rev=1616428&r1=1616427&r2=1616428&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java Thu Aug 7 07:38:23 2014 @@ -1005,7 +1005,7 @@ public final class HttpServer2 implement String remoteUser = request.getRemoteUser(); if (remoteUser == null) { - response.sendError(HttpServletResponse.SC_UNAUTHORIZED, + response.sendError(HttpServletResponse.SC_FORBIDDEN, "Unauthenticated users are not " + "authorized to access this page."); return false; @@ -1013,7 +1013,7 @@ public final class HttpServer2 implement if (servletContext.getAttribute(ADMINS_ACL) != null && !userHasAdministratorAccess(servletContext, remoteUser)) { - response.sendError(HttpServletResponse.SC_UNAUTHORIZED, "User " + response.sendError(HttpServletResponse.SC_FORBIDDEN, "User " + remoteUser + " is unauthorized to access this page."); return false; } Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java?rev=1616428&r1=1616427&r2=1616428&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java Thu Aug 7 07:38:23 2014 @@ -33,6 +33,7 @@ import org.apache.hadoop.classification. import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.fs.HardLink; import org.apache.hadoop.io.SecureIOUtils.AlreadyExistsException; import org.apache.hadoop.util.NativeCodeLoader; import org.apache.hadoop.util.Shell; @@ -823,6 +824,14 @@ public class NativeIO { } } + public static void link(File src, File dst) throws IOException { + if (!nativeLoaded) { + HardLink.createHardLink(src, dst); + } else { + link0(src.getAbsolutePath(), dst.getAbsolutePath()); + } + } + /** * A version of renameTo that throws a descriptive exception when it fails. * @@ -833,4 +842,7 @@ public class NativeIO { */ private static native void renameTo0(String src, String dst) throws NativeIOException; + + private static native void link0(String src, String dst) + throws NativeIOException; } Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java?rev=1616428&r1=1616427&r2=1616428&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java Thu Aug 7 07:38:23 2014 @@ -143,6 +143,12 @@ public class JMXJsonServlet extends Http jsonFactory = new JsonFactory(); } + protected boolean isInstrumentationAccessAllowed(HttpServletRequest request, + HttpServletResponse response) throws IOException { + return HttpServer2.isInstrumentationAccessAllowed(getServletContext(), + request, response); + } + /** * Process a GET request for the specified resource. * @@ -154,8 +160,7 @@ public class JMXJsonServlet extends Http @Override public void doGet(HttpServletRequest request, HttpServletResponse response) { try { - if (!HttpServer2.isInstrumentationAccessAllowed(getServletContext(), - request, response)) { + if (!isInstrumentationAccessAllowed(request, response)) { return; } JsonGenerator jg = null; Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java?rev=1616428&r1=1616427&r2=1616428&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java Thu Aug 7 07:38:23 2014 @@ -312,15 +312,15 @@ public class LdapGroupsMapping useSsl = conf.getBoolean(LDAP_USE_SSL_KEY, LDAP_USE_SSL_DEFAULT); keystore = conf.get(LDAP_KEYSTORE_KEY, LDAP_KEYSTORE_DEFAULT); - keystorePass = - conf.get(LDAP_KEYSTORE_PASSWORD_KEY, LDAP_KEYSTORE_PASSWORD_DEFAULT); + keystorePass = getPassword(conf, LDAP_KEYSTORE_PASSWORD_KEY, + LDAP_KEYSTORE_PASSWORD_DEFAULT); if (keystorePass.isEmpty()) { keystorePass = extractPassword(conf.get(LDAP_KEYSTORE_PASSWORD_FILE_KEY, LDAP_KEYSTORE_PASSWORD_FILE_DEFAULT)); } bindUser = conf.get(BIND_USER_KEY, BIND_USER_DEFAULT); - bindPassword = conf.get(BIND_PASSWORD_KEY, BIND_PASSWORD_DEFAULT); + bindPassword = getPassword(conf, BIND_PASSWORD_KEY, BIND_PASSWORD_DEFAULT); if (bindPassword.isEmpty()) { bindPassword = extractPassword( conf.get(BIND_PASSWORD_FILE_KEY, BIND_PASSWORD_FILE_DEFAULT)); @@ -341,7 +341,25 @@ public class LdapGroupsMapping this.conf = conf; } - + + String getPassword(Configuration conf, String alias, String defaultPass) { + String password = null; + try { + char[] passchars = conf.getPassword(alias); + if (passchars != null) { + password = new String(passchars); + } + else { + password = defaultPass; + } + } + catch (IOException ioe) { + LOG.warn("Exception while trying to password for alias " + alias + ": " + + ioe.getMessage()); + } + return password; + } + String extractPassword(String pwFile) { if (pwFile.isEmpty()) { // If there is no password file defined, we'll assume that we should do Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java?rev=1616428&r1=1616427&r2=1616428&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java Thu Aug 7 07:38:23 2014 @@ -77,7 +77,8 @@ public class SecurityUtil { * For use only by tests and initialization */ @InterfaceAudience.Private - static void setTokenServiceUseIp(boolean flag) { + @VisibleForTesting + public static void setTokenServiceUseIp(boolean flag) { useIpForTokenService = flag; hostResolver = !useIpForTokenService ? new QualifiedHostResolver() Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialShell.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialShell.java?rev=1616428&r1=1616427&r2=1616428&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialShell.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialShell.java Thu Aug 7 07:38:23 2014 @@ -67,11 +67,11 @@ public class CredentialShell extends Con if (command.validate()) { command.execute(); } else { - exitCode = -1; + exitCode = 1; } } catch (Exception e) { e.printStackTrace(err); - return -1; + return 1; } return exitCode; } @@ -79,47 +79,54 @@ public class CredentialShell extends Con /** * Parse the command line arguments and initialize the data * 
-   * % hadoop alias create alias [--provider providerPath]
-   * % hadoop alias list [-provider providerPath]
-   * % hadoop alias delete alias [--provider providerPath] [-i]
+   * % hadoop credential create alias [-provider providerPath]
+   * % hadoop credential list [-provider providerPath]
+   * % hadoop credential delete alias [-provider providerPath] [-i]
    *
* @param args - * @return + * @return 0 if the argument(s) were recognized, 1 otherwise * @throws IOException */ - private int init(String[] args) throws IOException { + protected int init(String[] args) throws IOException { + // no args should print the help message + if (0 == args.length) { + printCredShellUsage(); + ToolRunner.printGenericCommandUsage(System.err); + return 1; + } + for (int i = 0; i < args.length; i++) { // parse command line if (args[i].equals("create")) { String alias = args[++i]; command = new CreateCommand(alias); - if (alias.equals("--help")) { + if (alias.equals("-help")) { printCredShellUsage(); - return -1; + return 0; } } else if (args[i].equals("delete")) { String alias = args[++i]; command = new DeleteCommand(alias); - if (alias.equals("--help")) { + if (alias.equals("-help")) { printCredShellUsage(); - return -1; + return 0; } } else if (args[i].equals("list")) { command = new ListCommand(); - } else if (args[i].equals("--provider")) { + } else if (args[i].equals("-provider")) { userSuppliedProvider = true; getConf().set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, args[++i]); - } else if (args[i].equals("-i") || (args[i].equals("--interactive"))) { + } else if (args[i].equals("-i") || (args[i].equals("-interactive"))) { interactive = true; - } else if (args[i].equals("-v") || (args[i].equals("--value"))) { + } else if (args[i].equals("-v") || (args[i].equals("-value"))) { value = args[++i]; - } else if (args[i].equals("--help")) { + } else if (args[i].equals("-help")) { printCredShellUsage(); - return -1; + return 0; } else { printCredShellUsage(); ToolRunner.printGenericCommandUsage(System.err); - return -1; + return 1; } } return 0; @@ -188,20 +195,20 @@ public class CredentialShell extends Con } private class ListCommand extends Command { - public static final String USAGE = "list [--provider] [--help]"; + public static final String USAGE = "list [-provider] [-help]"; public static final String DESC = "The list subcommand displays the aliases contained within \n" + "a particular provider - as configured in core-site.xml or " + - "indicated\nthrough the --provider argument."; + "indicated\nthrough the -provider argument."; public boolean validate() { boolean rc = true; provider = getCredentialProvider(); if (provider == null) { out.println("There are no non-transient CredentialProviders configured.\n" - + "Consider using the --provider option to indicate the provider\n" + + "Consider using the -provider option to indicate the provider\n" + "to use. If you want to list a transient provider then you\n" - + "you MUST use the --provider argument."); + + "you MUST use the -provider argument."); rc = false; } return rc; @@ -229,11 +236,11 @@ public class CredentialShell extends Con } private class DeleteCommand extends Command { - public static final String USAGE = "delete [--provider] [--help]"; + public static final String USAGE = "delete [-provider] [-help]"; public static final String DESC = "The delete subcommand deletes the credenital\n" + "specified as the argument from within the provider\n" + - "indicated through the --provider argument"; + "indicated through the -provider argument"; String alias = null; boolean cont = true; @@ -248,13 +255,13 @@ public class CredentialShell extends Con if (provider == null) { out.println("There are no valid CredentialProviders configured.\n" + "Nothing will be deleted.\n" - + "Consider using the --provider option to indicate the provider" + + "Consider using the -provider option to indicate the provider" + " to use."); return false; } if (alias == null) { out.println("There is no alias specified. Please provide the" + - "mandatory . See the usage description with --help."); + "mandatory . See the usage description with -help."); return false; } if (interactive) { @@ -299,11 +306,11 @@ public class CredentialShell extends Con } private class CreateCommand extends Command { - public static final String USAGE = "create [--provider] [--help]"; + public static final String USAGE = "create [-provider] [-help]"; public static final String DESC = "The create subcommand creates a new credential for the name specified\n" + "as the argument within the provider indicated through\n" + - "the --provider argument."; + "the -provider argument."; String alias = null; @@ -317,13 +324,13 @@ public class CredentialShell extends Con if (provider == null) { out.println("There are no valid CredentialProviders configured." + "\nCredential will not be created.\n" - + "Consider using the --provider option to indicate the provider" + + + "Consider using the -provider option to indicate the provider" + " to use."); rc = false; } if (alias == null) { out.println("There is no alias specified. Please provide the" + - "mandatory . See the usage description with --help."); + "mandatory . See the usage description with -help."); rc = false; } return rc; Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java?rev=1616428&r1=1616427&r2=1616428&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java Thu Aug 7 07:38:23 2014 @@ -150,7 +150,7 @@ public class FileBasedKeyStoresFactory i } String passwordProperty = resolvePropertyName(mode, SSL_KEYSTORE_PASSWORD_TPL_KEY); - String keystorePassword = conf.get(passwordProperty, ""); + String keystorePassword = getPassword(conf, passwordProperty, ""); if (keystorePassword.isEmpty()) { throw new GeneralSecurityException("The property '" + passwordProperty + "' has not been set in the ssl configuration file."); @@ -160,7 +160,8 @@ public class FileBasedKeyStoresFactory i // Key password defaults to the same value as store password for // compatibility with legacy configurations that did not use a separate // configuration property for key password. - keystoreKeyPassword = conf.get(keyPasswordProperty, keystorePassword); + keystoreKeyPassword = getPassword( + conf, keyPasswordProperty, keystorePassword); LOG.debug(mode.toString() + " KeyStore: " + keystoreLocation); InputStream is = new FileInputStream(keystoreLocation); @@ -191,7 +192,7 @@ public class FileBasedKeyStoresFactory i if (!truststoreLocation.isEmpty()) { String passwordProperty = resolvePropertyName(mode, SSL_TRUSTSTORE_PASSWORD_TPL_KEY); - String truststorePassword = conf.get(passwordProperty, ""); + String truststorePassword = getPassword(conf, passwordProperty, ""); if (truststorePassword.isEmpty()) { throw new GeneralSecurityException("The property '" + passwordProperty + "' has not been set in the ssl configuration file."); @@ -217,6 +218,21 @@ public class FileBasedKeyStoresFactory i } } + String getPassword(Configuration conf, String alias, String defaultPass) { + String password = defaultPass; + try { + char[] passchars = conf.getPassword(alias); + if (passchars != null) { + password = new String(passchars); + } + } + catch (IOException ioe) { + LOG.warn("Exception while trying to get password for alias " + alias + + ": " + ioe.getMessage()); + } + return password; + } + /** * Releases any resources being used. */ Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java?rev=1616428&r1=1616427&r2=1616428&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java Thu Aug 7 07:38:23 2014 @@ -78,6 +78,20 @@ public class DiskChecker { (mkdirsWithExistsCheck(new File(parent)) && (canonDir.mkdir() || canonDir.exists())); } + + /** + * Recurse down a directory tree, checking all child directories. + * @param dir + * @throws DiskErrorException + */ + public static void checkDirs(File dir) throws DiskErrorException { + checkDir(dir); + for (File child : dir.listFiles()) { + if (child.isDirectory()) { + checkDirs(child); + } + } + } /** * Create the directory if it doesn't exist and check that dir is readable, Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c?rev=1616428&r1=1616427&r2=1616428&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c (original) +++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c Thu Aug 7 07:38:23 2014 @@ -1054,6 +1054,43 @@ done: #endif } +JNIEXPORT void JNICALL +Java_org_apache_hadoop_io_nativeio_NativeIO_link0(JNIEnv *env, +jclass clazz, jstring jsrc, jstring jdst) +{ +#ifdef UNIX + const char *src = NULL, *dst = NULL; + + src = (*env)->GetStringUTFChars(env, jsrc, NULL); + if (!src) goto done; // exception was thrown + dst = (*env)->GetStringUTFChars(env, jdst, NULL); + if (!dst) goto done; // exception was thrown + if (link(src, dst)) { + throw_ioe(env, errno); + } + +done: + if (src) (*env)->ReleaseStringUTFChars(env, jsrc, src); + if (dst) (*env)->ReleaseStringUTFChars(env, jdst, dst); +#endif + +#ifdef WINDOWS + LPCTSTR src = NULL, dst = NULL; + + src = (LPCTSTR) (*env)->GetStringChars(env, jsrc, NULL); + if (!src) goto done; // exception was thrown + dst = (LPCTSTR) (*env)->GetStringChars(env, jdst, NULL); + if (!dst) goto done; // exception was thrown + if (!CreateHardLink(dst, src, NULL)) { + throw_ioe(env, GetLastError()); + } + +done: + if (src) (*env)->ReleaseStringChars(env, jsrc, src); + if (dst) (*env)->ReleaseStringChars(env, jdst, dst); +#endif +} + JNIEXPORT jlong JNICALL Java_org_apache_hadoop_io_nativeio_NativeIO_getMemlockLimit0( JNIEnv *env, jclass clazz)