Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 0614E10605 for ; Wed, 19 Mar 2014 20:19:34 +0000 (UTC) Received: (qmail 95946 invoked by uid 500); 19 Mar 2014 20:19:32 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 95902 invoked by uid 500); 19 Mar 2014 20:19:31 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 95894 invoked by uid 99); 19 Mar 2014 20:19:31 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 19 Mar 2014 20:19:31 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 19 Mar 2014 20:19:27 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 4D1C623889CB; Wed, 19 Mar 2014 20:19:05 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1579384 - in /hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common: ./ src/main/java/org/apache/hadoop/fs/ src/main/java/org/apache/hadoop/ipc/ src/main/java/org/apache/hadoop/security/ src/main/resources/ src/test/java/or... Date: Wed, 19 Mar 2014 20:19:04 -0000 To: common-commits@hadoop.apache.org From: arp@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20140319201905.4D1C623889CB@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: arp Date: Wed Mar 19 20:19:04 2014 New Revision: 1579384 URL: http://svn.apache.org/r1579384 Log: HADOOP-10221. Merging r1579383 from branch-2 to branch-2.4. Modified: hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcServer.java hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java Modified: hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1579384&r1=1579383&r2=1579384&view=diff ============================================================================== --- hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/CHANGES.txt Wed Mar 19 20:19:04 2014 @@ -52,6 +52,10 @@ Release 2.4.0 - UNRELEASED HADOOP-10386. Log proxy hostname in various exceptions being thrown in a HA setup. (wheat9) + HADOOP-10221. Add a plugin to specify SaslProperties for RPC protocol + based on connection properties. (Benoy Antony and Daryn Sharp via + Arpit Agarwal) + OPTIMIZATIONS BUG FIXES Modified: hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java?rev=1579384&r1=1579383&r2=1579384&view=diff ============================================================================== --- hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java (original) +++ hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java Wed Mar 19 20:19:04 2014 @@ -282,5 +282,11 @@ public class CommonConfigurationKeysPubl public static final String HTTP_POLICY_HTTP_ONLY = "HTTP_ONLY"; @Deprecated public static final String HTTP_POLICY_HTTPS_ONLY = "HTTPS_ONLY"; + /** See core-default.xml */ + public static final String HADOOP_RPC_PROTECTION = + "hadoop.rpc.protection"; + /** Class to override Sasl Properties for a connection */ + public static final String HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS = + "hadoop.security.saslproperties.resolver.class"; } Modified: hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java?rev=1579384&r1=1579383&r2=1579384&view=diff ============================================================================== --- hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java (original) +++ hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java Wed Mar 19 20:19:04 2014 @@ -729,6 +729,7 @@ public class Client { // for testing remoteId.saslQop = (String)saslRpcClient.getNegotiatedProperty(Sasl.QOP); + LOG.debug("Negotiated QOP is :" + remoteId.saslQop); } else if (UserGroupInformation.isSecurityEnabled() && !fallbackAllowed) { throw new IOException("Server asks us to fall back to SIMPLE " + Modified: hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java?rev=1579384&r1=1579383&r2=1579384&view=diff ============================================================================== --- hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java (original) +++ hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java Wed Mar 19 20:19:04 2014 @@ -97,6 +97,7 @@ import org.apache.hadoop.ipc.protobuf.Rp import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcSaslProto.SaslState; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.AccessControlException; +import org.apache.hadoop.security.SaslPropertiesResolver; import org.apache.hadoop.security.SaslRpcServer; import org.apache.hadoop.security.SaslRpcServer.AuthMethod; import org.apache.hadoop.security.SecurityUtil; @@ -357,6 +358,7 @@ public abstract class Server { private Configuration conf; private String portRangeConfig = null; private SecretManager secretManager; + private SaslPropertiesResolver saslPropsResolver; private ServiceAuthorizationManager serviceAuthorizationManager = new ServiceAuthorizationManager(); private int maxQueueSize; @@ -1566,7 +1568,9 @@ public abstract class Server { private SaslServer createSaslServer(AuthMethod authMethod) throws IOException, InterruptedException { - return new SaslRpcServer(authMethod).create(this, secretManager); + final Map saslProps = + saslPropsResolver.getServerProperties(addr); + return new SaslRpcServer(authMethod).create(this ,saslProps, secretManager); } /** @@ -2168,6 +2172,7 @@ public abstract class Server { if (secretManager != null || UserGroupInformation.isSecurityEnabled()) { SaslRpcServer.init(conf); + saslPropsResolver = SaslPropertiesResolver.getInstance(conf); } this.exceptionsHandler.addTerseExceptions(StandbyException.class); Modified: hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java?rev=1579384&r1=1579383&r2=1579384&view=diff ============================================================================== --- hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java (original) +++ hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java Wed Mar 19 20:19:04 2014 @@ -89,6 +89,7 @@ public class SaslRpcClient { private final Configuration conf; private SaslClient saslClient; + private SaslPropertiesResolver saslPropsResolver; private AuthMethod authMethod; private static final RpcRequestHeaderProto saslHeader = ProtoUtil @@ -112,6 +113,7 @@ public class SaslRpcClient { this.protocol = protocol; this.serverAddr = serverAddr; this.conf = conf; + this.saslPropsResolver = SaslPropertiesResolver.getInstance(conf); } @VisibleForTesting @@ -207,7 +209,8 @@ public class SaslRpcClient { // if necessary, auth types below will verify they are valid final String saslProtocol = authType.getProtocol(); final String saslServerName = authType.getServerId(); - Map saslProperties = SaslRpcServer.SASL_PROPS; + Map saslProperties = + saslPropsResolver.getClientProperties(serverAddr.getAddress()); CallbackHandler saslCallback = null; final AuthMethod method = AuthMethod.valueOf(authType.getMethod()); Modified: hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcServer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcServer.java?rev=1579384&r1=1579383&r2=1579384&view=diff ============================================================================== --- hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcServer.java (original) +++ hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcServer.java Wed Mar 19 20:19:04 2014 @@ -70,6 +70,7 @@ public class SaslRpcServer { public static final Map SASL_PROPS = new TreeMap(); private static SaslServerFactory saslFactory; + private static SaslPropertiesResolver resolver; public static enum QualityOfProtection { AUTHENTICATION("auth"), @@ -129,7 +130,8 @@ public class SaslRpcServer { @InterfaceAudience.Private @InterfaceStability.Unstable - public SaslServer create(Connection connection, + public SaslServer create(final Connection connection, + final Map saslProperties, SecretManager secretManager ) throws IOException, InterruptedException { UserGroupInformation ugi = null; @@ -162,12 +164,12 @@ public class SaslRpcServer { @Override public SaslServer run() throws SaslException { return saslFactory.createSaslServer(mechanism, protocol, serverId, - SaslRpcServer.SASL_PROPS, callback); + saslProperties, callback); } }); } else { saslServer = saslFactory.createSaslServer(mechanism, protocol, serverId, - SaslRpcServer.SASL_PROPS, callback); + saslProperties, callback); } if (saslServer == null) { throw new AccessControlException( @@ -180,17 +182,10 @@ public class SaslRpcServer { } public static void init(Configuration conf) { - String[] qop = conf.getStrings("hadoop.rpc.protection", - QualityOfProtection.AUTHENTICATION.toString()); - - for (int i=0; i < qop.length; i++) { - qop[i] = QualityOfProtection.valueOf(qop[i].toUpperCase()).getSaslQop(); - } - - SASL_PROPS.put(Sasl.QOP, StringUtils.join(",", qop)); - SASL_PROPS.put(Sasl.SERVER_AUTH, "true"); Security.addProvider(new SaslPlainServer.SecurityProvider()); - saslFactory = new FastSaslServerFactory(SASL_PROPS); + // passing null so factory is populated with all possibilities. the + // properties passed when instantiating a server are what really matter + saslFactory = new FastSaslServerFactory(null); } static String encodeIdentifier(byte[] identifier) { Modified: hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml?rev=1579384&r1=1579383&r2=1579384&view=diff ============================================================================== --- hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml (original) +++ hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml Wed Mar 19 20:19:04 2014 @@ -261,6 +261,19 @@ authentication means authentication only and no integrity or privacy; integrity implies authentication and integrity are enabled; and privacy implies all of authentication, integrity and privacy are enabled. + hadoop.security.saslproperties.resolver.class can be used to override + the hadoop.rpc.protection for a connection at the server side. + + + + + hadoop.security.saslproperties.resolver.class + + SaslPropertiesResolver used to resolve the QOP used for a + connection. If not specified, the full set of values specified in + hadoop.rpc.protection is used while determining the QOP used for the + connection. If a class is specified, then the QOP values returned by + the class will be used while determining the QOP used for the connection. Modified: hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java?rev=1579384&r1=1579383&r2=1579384&view=diff ============================================================================== --- hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java (original) +++ hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java Wed Mar 19 20:19:04 2014 @@ -19,6 +19,7 @@ package org.apache.hadoop.ipc; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_RPC_PROTECTION; import static org.apache.hadoop.security.SaslRpcServer.AuthMethod.KERBEROS; import static org.apache.hadoop.security.SaslRpcServer.AuthMethod.SIMPLE; import static org.apache.hadoop.security.SaslRpcServer.AuthMethod.TOKEN; @@ -33,11 +34,14 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; import java.lang.annotation.Annotation; +import java.net.InetAddress; import java.net.InetSocketAddress; import java.security.PrivilegedExceptionAction; import java.security.Security; import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; +import java.util.Map; import java.util.Set; import java.util.regex.Pattern; @@ -65,6 +69,7 @@ import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.KerberosInfo; import org.apache.hadoop.security.SaslInputStream; import org.apache.hadoop.security.SaslPlainServer; +import org.apache.hadoop.security.SaslPropertiesResolver; import org.apache.hadoop.security.SaslRpcClient; import org.apache.hadoop.security.SaslRpcServer; import org.apache.hadoop.security.SaslRpcServer.AuthMethod; @@ -94,21 +99,29 @@ public class TestSaslRPC { public static Collection data() { Collection params = new ArrayList(); for (QualityOfProtection qop : QualityOfProtection.values()) { - params.add(new Object[]{ new QualityOfProtection[]{qop},qop }); + params.add(new Object[]{ new QualityOfProtection[]{qop},qop, null }); } params.add(new Object[]{ new QualityOfProtection[]{ QualityOfProtection.PRIVACY,QualityOfProtection.AUTHENTICATION }, - QualityOfProtection.PRIVACY }); + QualityOfProtection.PRIVACY, null}); + params.add(new Object[]{ new QualityOfProtection[]{ + QualityOfProtection.PRIVACY,QualityOfProtection.AUTHENTICATION }, + QualityOfProtection.AUTHENTICATION , + "org.apache.hadoop.ipc.TestSaslRPC$AuthSaslPropertiesResolver" }); + return params; } QualityOfProtection[] qop; QualityOfProtection expectedQop; + String saslPropertiesResolver ; public TestSaslRPC(QualityOfProtection[] qop, - QualityOfProtection expectedQop) { + QualityOfProtection expectedQop, + String saslPropertiesResolver) { this.qop=qop; this.expectedQop = expectedQop; + this.saslPropertiesResolver = saslPropertiesResolver; } private static final String ADDRESS = "0.0.0.0"; @@ -153,7 +166,11 @@ public class TestSaslRPC { // the specific tests for kerberos will enable kerberos. forcing it // for all tests will cause tests to fail if the user has a TGT conf.set(HADOOP_SECURITY_AUTHENTICATION, SIMPLE.toString()); - conf.set("hadoop.rpc.protection", getQOPNames(qop)); + conf.set(HADOOP_RPC_PROTECTION, getQOPNames(qop)); + if (saslPropertiesResolver != null){ + conf.set(CommonConfigurationKeys.HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS, + saslPropertiesResolver); + } UserGroupInformation.setConfiguration(conf); enableSecretManager = null; forceSecretManager = null; @@ -964,6 +981,19 @@ public class TestSaslRPC { } } + /* + * Class used to test overriding QOP values using SaslPropertiesResolver + */ + static class AuthSaslPropertiesResolver extends SaslPropertiesResolver{ + + @Override + public Map getServerProperties(InetAddress address) { + Map newPropertes = new HashMap(getDefaultProperties()); + newPropertes.put(Sasl.QOP, QualityOfProtection.AUTHENTICATION.getSaslQop()); + return newPropertes; + } + } + public static void main(String[] args) throws Exception { System.out.println("Testing Kerberos authentication over RPC"); if (args.length != 2) {