Author: ddas
Date: Wed Feb 3 02:57:35 2010
New Revision: 905873
URL: http://svn.apache.org/viewvc?rev=905873&view=rev
Log:
HDFS-933. Adds Delegation token based authentication in the NameNode. Contributed by Kan Zhang.
Added:
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/security/token/DelegationTokenSelector.java
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/security/TestClientProtocolWithDelegationToken.java
Modified:
hadoop/hdfs/trunk/CHANGES.txt
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/security/token/DelegationTokenSecretManager.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/protocol/InterDatanodeProtocol.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java
Modified: hadoop/hdfs/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=905873&r1=905872&r2=905873&view=diff
==============================================================================
--- hadoop/hdfs/trunk/CHANGES.txt (original)
+++ hadoop/hdfs/trunk/CHANGES.txt Wed Feb 3 02:57:35 2010
@@ -15,6 +15,9 @@
HDFS-222. Support for concatenating of files into a single file
without copying. (Boris Shkolnik via hairong)
+ HDFS-933. Adds Delegation token based authentication in the NameNode.
+ (Kan Zhang via ddas)
+
IMPROVEMENTS
HDFS-703. Replace current fault injection implementation with one
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java?rev=905873&r1=905872&r2=905873&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java Wed Feb
3 02:57:35 2010
@@ -27,14 +27,18 @@
import org.apache.hadoop.fs.FsServerDefaults;
import org.apache.hadoop.fs.Options;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.protocol.FSConstants.UpgradeAction;
import org.apache.hadoop.hdfs.security.token.DelegationTokenIdentifier;
+import org.apache.hadoop.hdfs.security.token.DelegationTokenSelector;
import org.apache.hadoop.hdfs.server.common.UpgradeStatusReport;
import org.apache.hadoop.io.EnumSetWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.ipc.VersionedProtocol;
import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.KerberosInfo;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenInfo;
/**********************************************************************
* ClientProtocol is used by user code via
@@ -43,6 +47,8 @@
* as well as open/close file streams, etc.
*
**********************************************************************/
+@KerberosInfo(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY)
+@TokenInfo(DelegationTokenSelector.class)
public interface ClientProtocol extends VersionedProtocol {
/**
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/security/token/DelegationTokenSecretManager.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/security/token/DelegationTokenSecretManager.java?rev=905873&r1=905872&r2=905873&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/security/token/DelegationTokenSecretManager.java
(original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/security/token/DelegationTokenSecretManager.java
Wed Feb 3 02:57:35 2010
@@ -268,6 +268,14 @@
return SecretManager.createSecretKey(key);
}
+ /**
+ * Create an empty delegation token identifier
+ * @return a newly created empty delegation token identifier
+ */
+ @Override
+ public DelegationTokenIdentifier createIdentifier() {
+ return new DelegationTokenIdentifier();
+ }
/** Utility class to encapsulate a token's renew date and password. */
private static class DelegationTokenInformation {
Added: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/security/token/DelegationTokenSelector.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/security/token/DelegationTokenSelector.java?rev=905873&view=auto
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/security/token/DelegationTokenSelector.java
(added)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/security/token/DelegationTokenSelector.java
Wed Feb 3 02:57:35 2010
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.security.token;
+
+import java.util.Collection;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.security.token.TokenSelector;
+
+/**
+ * Look through tokens to find the first delegation token that matches the
+ * service and return it.
+ */
+@InterfaceAudience.Private
+public class DelegationTokenSelector implements
+ TokenSelector<DelegationTokenIdentifier> {
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public Token<DelegationTokenIdentifier> selectToken(Text service,
+ Collection<Token<? extends TokenIdentifier>> tokens) {
+ if (service == null) {
+ return null;
+ }
+ for (Token<? extends TokenIdentifier> token : tokens) {
+ if (DelegationTokenIdentifier.KIND_NAME.equals(token.getKind())
+ && service.equals(token.getService())) {
+ return (Token<DelegationTokenIdentifier>) token;
+ }
+ }
+ return null;
+ }
+}
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=905873&r1=905872&r2=905873&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Wed
Feb 3 02:57:35 2010
@@ -284,13 +284,13 @@
accessTokenHandler = new AccessTokenHandler(true,
accessKeyUpdateInterval, accessTokenLifetime);
}
- dtSecretManager.startThreads();
}
/**
* Activate FSNamesystem daemons.
*/
void activate(Configuration conf) throws IOException {
+ dtSecretManager.startThreads();
setBlockTotal();
blockManager.activate();
this.hbthread = new Daemon(new HeartbeatMonitor());
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=905873&r1=905872&r2=905873&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java Wed Feb
3 02:57:35 2010
@@ -295,16 +295,17 @@
ServiceAuthorizationManager.refresh(conf, new HDFSPolicyProvider());
}
+ NameNode.initMetrics(conf, this.getRole());
+ loadNamesystem(conf);
// create rpc server
- this.server = RPC.getServer(this, socAddr.getHostName(), socAddr.getPort(),
- handlerCount, false, conf);
+ this.server = RPC.getServer(this.getClass(), this, socAddr.getHostName(),
+ socAddr.getPort(), handlerCount, false, conf, namesystem
+ .getDelegationTokenSecretManager());
// The rpc-server port can be ephemeral... ensure we have the correct info
this.rpcAddress = this.server.getListenerAddress();
setRpcServerAddress(conf);
- NameNode.initMetrics(conf, this.getRole());
- loadNamesystem(conf);
activate(conf);
LOG.info(getRole() + " up at: " + rpcAddress);
}
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java?rev=905873&r1=905872&r2=905873&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java
(original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java
Wed Feb 3 02:57:35 2010
@@ -20,10 +20,12 @@
import java.io.*;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.ipc.VersionedProtocol;
+import org.apache.hadoop.security.KerberosInfo;
/**********************************************************************
* Protocol that a DFS datanode uses to communicate with the NameNode.
@@ -33,6 +35,7 @@
* returning values from these functions.
*
**********************************************************************/
+@KerberosInfo(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY)
public interface DatanodeProtocol extends VersionedProtocol {
/**
* 23: nextGenerationStamp() removed.
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/protocol/InterDatanodeProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/protocol/InterDatanodeProtocol.java?rev=905873&r1=905872&r2=905873&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/protocol/InterDatanodeProtocol.java
(original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/protocol/InterDatanodeProtocol.java
Wed Feb 3 02:57:35 2010
@@ -22,12 +22,15 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock;
import org.apache.hadoop.ipc.VersionedProtocol;
+import org.apache.hadoop.security.KerberosInfo;
/** An inter-datanode protocol for updating generation stamp
*/
+@KerberosInfo(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY)
public interface InterDatanodeProtocol extends VersionedProtocol {
public static final Log LOG = LogFactory.getLog(InterDatanodeProtocol.class);
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java?rev=905873&r1=905872&r2=905873&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java
(original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java
Wed Feb 3 02:57:35 2010
@@ -20,15 +20,18 @@
import java.io.IOException;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.security.ExportedAccessKeys;
import org.apache.hadoop.hdfs.server.namenode.CheckpointSignature;
import org.apache.hadoop.ipc.VersionedProtocol;
+import org.apache.hadoop.security.KerberosInfo;
/*****************************************************************************
* Protocol that a secondary NameNode uses to communicate with the NameNode.
* It's used to get part of the name node state
*****************************************************************************/
+@KerberosInfo(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY)
public interface NamenodeProtocol extends VersionedProtocol {
/**
* Compared to the previous version the following changes have been introduced:
Added: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/security/TestClientProtocolWithDelegationToken.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/security/TestClientProtocolWithDelegationToken.java?rev=905873&view=auto
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/security/TestClientProtocolWithDelegationToken.java
(added)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/security/TestClientProtocolWithDelegationToken.java
Wed Feb 3 02:57:35 2010
@@ -0,0 +1,122 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.security;
+
+import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION;
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.net.InetSocketAddress;
+import java.security.PrivilegedExceptionAction;
+
+import org.apache.commons.logging.*;
+import org.apache.commons.logging.impl.Log4JLogger;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+
+import org.apache.hadoop.ipc.Client;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.security.token.DelegationTokenIdentifier;
+import org.apache.hadoop.hdfs.security.token.DelegationTokenSecretManager;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.SaslInputStream;
+import org.apache.hadoop.security.SaslRpcClient;
+import org.apache.hadoop.security.SaslRpcServer;
+import org.apache.hadoop.security.UserGroupInformation;
+
+import org.apache.log4j.Level;
+import org.junit.Test;
+
+/** Unit tests for using Delegation Token over RPC. */
+public class TestClientProtocolWithDelegationToken {
+ private static final String ADDRESS = "0.0.0.0";
+
+ public static final Log LOG = LogFactory
+ .getLog(TestClientProtocolWithDelegationToken.class);
+
+ private static Configuration conf;
+ static {
+ conf = new Configuration();
+ conf.set(HADOOP_SECURITY_AUTHENTICATION, "kerberos");
+ UserGroupInformation.setConfiguration(conf);
+ }
+
+ static {
+ ((Log4JLogger) Client.LOG).getLogger().setLevel(Level.ALL);
+ ((Log4JLogger) Server.LOG).getLogger().setLevel(Level.ALL);
+ ((Log4JLogger) SaslRpcClient.LOG).getLogger().setLevel(Level.ALL);
+ ((Log4JLogger) SaslRpcServer.LOG).getLogger().setLevel(Level.ALL);
+ ((Log4JLogger) SaslInputStream.LOG).getLogger().setLevel(Level.ALL);
+ }
+
+ @Test
+ public void testDelegationTokenRpc() throws Exception {
+ ClientProtocol mockNN = mock(ClientProtocol.class);
+ when(mockNN.getProtocolVersion(anyString(), anyLong())).thenReturn(
+ ClientProtocol.versionID);
+ DelegationTokenSecretManager sm = new DelegationTokenSecretManager(
+ DFSConfigKeys.DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT,
+ DFSConfigKeys.DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT,
+ DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT,
+ 3600000);
+ sm.startThreads();
+ final Server server = RPC.getServer(ClientProtocol.class, mockNN, ADDRESS,
+ 0, 5, true, conf, sm);
+
+ server.start();
+
+ final UserGroupInformation current = UserGroupInformation.getCurrentUser();
+ final InetSocketAddress addr = NetUtils.getConnectAddress(server);
+ String user = current.getUserName();
+ Text owner = new Text(user);
+ DelegationTokenIdentifier dtId = new DelegationTokenIdentifier(owner, owner);
+ Token<DelegationTokenIdentifier> token = new Token<DelegationTokenIdentifier>(
+ dtId, sm);
+ Text host = new Text(addr.getAddress().getHostAddress() + ":"
+ + addr.getPort());
+ token.setService(host);
+ LOG.info("Service IP address for token is " + host);
+ current.addToken(token);
+ current.doAs(new PrivilegedExceptionAction<Object>() {
+ @Override
+ public Object run() throws Exception {
+ ClientProtocol proxy = null;
+ try {
+ proxy = (ClientProtocol) RPC.getProxy(ClientProtocol.class,
+ ClientProtocol.versionID, addr, conf);
+ proxy.getServerDefaults();
+ } finally {
+ server.stop();
+ if (proxy != null) {
+ RPC.stopProxy(proxy);
+ }
+ }
+ return null;
+ }
+ });
+ }
+
+}
|