hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jgho...@apache.org
Subject svn commit: r1035508 - in /hadoop/hdfs/trunk: CHANGES.txt src/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java src/test/hdfs/org/apache/hadoop/tools/TestDelegationTokenFetcher.java
Date Tue, 16 Nov 2010 00:41:18 GMT
Author: jghoman
Date: Tue Nov 16 00:41:17 2010
New Revision: 1035508

URL: http://svn.apache.org/viewvc?rev=1035508&view=rev
Log:
HDFS-1187. Modify fetchdt to allow renewing and canceling token.  Contributed by Owen O'Malley
and Kan Zhang.

Modified:
    hadoop/hdfs/trunk/CHANGES.txt
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
    hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/tools/TestDelegationTokenFetcher.java

Modified: hadoop/hdfs/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=1035508&r1=1035507&r2=1035508&view=diff
==============================================================================
--- hadoop/hdfs/trunk/CHANGES.txt (original)
+++ hadoop/hdfs/trunk/CHANGES.txt Tue Nov 16 00:41:17 2010
@@ -188,6 +188,9 @@ Trunk (unreleased changes)
 
     HDFS-697. Enable asserts for tests by default. (eli)
 
+    HDFS-1187. Modify fetchdt to allow renewing and canceling token.
+    (Owen O'Malley and Kan Zhang via jghoman)
+
   OPTIMIZATIONS
 
     HDFS-1140. Speedup INode.getPathComponents. (Dmytro Molkov via shv)

Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java?rev=1035508&r1=1035507&r2=1035508&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java Tue
Nov 16 00:41:17 2010
@@ -18,37 +18,40 @@
 package org.apache.hadoop.hdfs.tools;
 
 import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
 import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.io.PrintStream;
 import java.net.HttpURLConnection;
-import java.net.InetAddress;
-import java.net.URI;
 import java.net.URL;
 import java.net.URLConnection;
 import java.security.PrivilegedExceptionAction;
+import java.util.Collection;
+import java.util.Date;
 
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Options;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager;
 import org.apache.hadoop.hdfs.server.namenode.CancelDelegationTokenServlet;
 import org.apache.hadoop.hdfs.server.namenode.GetDelegationTokenServlet;
 import org.apache.hadoop.hdfs.server.namenode.RenewDelegationTokenServlet;
 import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.io.Text;
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.GenericOptionsParser;
 
 /**
  * Fetch a DelegationToken from the current Namenode and store it in the
@@ -56,99 +59,153 @@ import org.apache.hadoop.util.StringUtil
  */
 @InterfaceAudience.Private
 public class DelegationTokenFetcher {
-  private static final String USAGE =
-    "fetchdt retrieves delegation tokens (optionally over http)\n" +
-    "and writes them to specified file.\n" +
-    "Usage: fetchdt [--webservice <namenode http addr>] <output filename>";
-  
-  private final DistributedFileSystem dfs;
-  private final UserGroupInformation ugi;
-  private final DataOutputStream out;
-  private final Configuration conf;
-  
   private static final Log LOG = 
     LogFactory.getLog(DelegationTokenFetcher.class);
+  private static final String WEBSERVICE = "webservice";
+  private static final String RENEWER = "renewer";
+  private static final String CANCEL = "cancel";
+  private static final String RENEW = "renew";
+  private static final String PRINT = "print";
 
   static {
     // Enable Kerberos sockets
     System.setProperty("https.cipherSuites", "TLS_KRB5_WITH_3DES_EDE_CBC_SHA");
   }
 
-  /**
-   * Command-line interface
-   */
-  public static void main(final String [] args) throws Exception {
-    // Login the current user
-    UserGroupInformation.getCurrentUser().doAs(new PrivilegedExceptionAction<Object>()
{
-      @Override
-      public Object run() throws Exception {
-        
-        if(args.length == 3 && "--webservice".equals(args[0])) {
-          getDTfromRemoteIntoFile(args[1], args[2]);
-          return null;
-        }
-        // avoid annoying mistake
-        if(args.length == 1 && "--webservice".equals(args[0])) {
-          System.out.println(USAGE);
-          return null;
-        }
-        if(args.length != 1 || args[0].isEmpty()) {
-          System.out.println(USAGE);
-          return null;
-        }
-        
-        DataOutputStream out = null;
-        
-        try {
-          Configuration conf = new HdfsConfiguration();
-          DistributedFileSystem dfs = (DistributedFileSystem) FileSystem.get(conf);
-          out = new DataOutputStream(new FileOutputStream(args[0]));
-          UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
-
-          new DelegationTokenFetcher(dfs, out, ugi, conf).go();
-          
-          out.flush();
-          System.out.println("Succesfully wrote token of size " + 
-              out.size() + " bytes to "+ args[0]);
-        } catch (IOException ioe) {
-          System.out.println("Exception encountered:\n" +
-              StringUtils.stringifyException(ioe));
-        } finally {
-          if(out != null) out.close();
-        }
-        return null;
-      }
-    });
-
-  }
-  
-  public DelegationTokenFetcher(DistributedFileSystem dfs, 
-      DataOutputStream out, UserGroupInformation ugi, Configuration conf) {
-    checkNotNull("dfs", dfs); this.dfs = dfs;
-    checkNotNull("out", out); this.out = out;
-    checkNotNull("ugi", ugi); this.ugi = ugi;
-    checkNotNull("conf",conf); this.conf = conf;
-  }
-  
-  private void checkNotNull(String s, Object o) {
-    if(o == null) throw new IllegalArgumentException(s + " cannot be null.");
+  private static void printUsage(PrintStream err) throws IOException {
+    err.println("fetchdt retrieves delegation tokens from the NameNode");
+    err.println();
+    err.println("fetchdt <opts> <token file>");
+    err.println("Options:");
+    err.println("  --webservice <url>  Url to contact NN on");
+    err.println("  --renewer <name>    Name of the delegation token renewer");
+    err.println("  --cancel            Cancel the delegation token");
+    err.println("  --renew             Renew the delegation token");
+    err.println("  --print             Print the delegation token");
+    err.println();
+    GenericOptionsParser.printGenericCommandUsage(err);
+    System.exit(1);
   }
 
-  public void go() throws IOException {
-    String fullName = ugi.getUserName();
-    String shortName = ugi.getShortUserName();
-    Token<DelegationTokenIdentifier> token = 
-      dfs.getDelegationToken(fullName);
+  private static Collection<Token<?>> readTokens(Path file, Configuration conf)
+      throws IOException {
+    Credentials creds = Credentials.readTokenStorageFile(file, conf);
+    return creds.getAllTokens();
+  }
     
-    // Reconstruct the ip:port of the Namenode
-    URI uri = FileSystem.getDefaultUri(conf);
-    String nnAddress = 
-      InetAddress.getByName(uri.getHost()).getHostAddress() + ":" + uri.getPort();
-    token.setService(new Text(nnAddress));
+  /**
+   * Command-line interface
+   */
+  public static void main(final String[] args) throws Exception {
+    final Configuration conf = new HdfsConfiguration();
+    Options fetcherOptions = new Options();
+    fetcherOptions.addOption(WEBSERVICE, true,
+        "HTTPS url to reach the NameNode at");
+    fetcherOptions.addOption(RENEWER, true,
+        "Name of the delegation token renewer");
+    fetcherOptions.addOption(CANCEL, false, "cancel the token");
+    fetcherOptions.addOption(RENEW, false, "renew the token");
+    fetcherOptions.addOption(PRINT, false, "print the token");
+    GenericOptionsParser parser = new GenericOptionsParser(conf,
+        fetcherOptions, args);
+    CommandLine cmd = parser.getCommandLine();
     
-    Credentials ts = new Credentials();
-    ts.addToken(new Text(shortName), token);
-    ts.writeTokenStorageToStream(out);
+    // get options
+    final String webUrl = cmd.hasOption(WEBSERVICE) ? cmd
+        .getOptionValue(WEBSERVICE) : null;
+    final String renewer = cmd.hasOption(RENEWER) ? 
+        cmd.getOptionValue(RENEWER) : null;
+    final boolean cancel = cmd.hasOption(CANCEL);
+    final boolean renew = cmd.hasOption(RENEW);
+    final boolean print = cmd.hasOption(PRINT);
+    String[] remaining = parser.getRemainingArgs();
+
+    // check option validity
+    if (cancel && renew || cancel && print || renew && print || cancel
&& renew
+        && print) {
+      System.err.println("ERROR: Only specify cancel, renew or print.");
+      printUsage(System.err);
+    }
+    if (remaining.length != 1 || remaining[0].charAt(0) == '-') {
+      System.err.println("ERROR: Must specify exacltly one token file");
+      printUsage(System.err);
+    }
+    // default to using the local file system
+    FileSystem local = FileSystem.getLocal(conf);
+    final Path tokenFile = new Path(local.getWorkingDirectory(), remaining[0]);
+
+    // Login the current user
+    UserGroupInformation.getCurrentUser().doAs(
+        new PrivilegedExceptionAction<Object>() {
+          @SuppressWarnings("unchecked")
+          @Override
+          public Object run() throws Exception {
+
+            if (print) {
+              DelegationTokenIdentifier id = new DelegationTokenSecretManager(
+                  0, 0, 0, 0, null).createIdentifier();
+              for (Token<?> token : readTokens(tokenFile, conf)) {
+                DataInputStream in = new DataInputStream(
+                    new ByteArrayInputStream(token.getIdentifier()));
+                id.readFields(in);
+                System.out.println("Token (" + id + ") for " + token.getService());
+              }
+              return null;
+            }
+            
+            if (webUrl != null) {
+              if (renew) {
+                long result;
+                for (Token<?> token : readTokens(tokenFile, conf)) {
+                  result = renewDelegationToken(webUrl,
+                      (Token<DelegationTokenIdentifier>) token);
+                  System.out.println("Renewed token via " + webUrl + " for "
+                      + token.getService() + " until: " + new Date(result));
+                }
+              } else if (cancel) {
+                for (Token<?> token : readTokens(tokenFile, conf)) {
+                  cancelDelegationToken(webUrl,
+                      (Token<DelegationTokenIdentifier>) token);
+                  System.out.println("Cancelled token via " + webUrl + " for "
+                      + token.getService());
+                }
+              } else {
+                Credentials creds = getDTfromRemote(webUrl, renewer);
+                creds.writeTokenStorageFile(tokenFile, conf);
+                for (Token<?> token : creds.getAllTokens()) {
+                  System.out.println("Fetched token via " + webUrl + " for "
+                      + token.getService() + " into " + tokenFile);
+                }
+              }
+            } else {
+              FileSystem fs = FileSystem.get(conf);
+              if (cancel) {
+                for (Token<?> token : readTokens(tokenFile, conf)) {
+                  ((DistributedFileSystem) fs)
+                      .cancelDelegationToken((Token<DelegationTokenIdentifier>) token);
+                  System.out.println("Cancelled token for "
+                      + token.getService());
+                }
+              } else if (renew) {
+                long result;
+                for (Token<?> token : readTokens(tokenFile, conf)) {
+                  result = ((DistributedFileSystem) fs)
+                      .renewDelegationToken((Token<DelegationTokenIdentifier>) token);
+                  System.out.println("Renewed token for " + token.getService()
+                      + " until: " + new Date(result));
+                }
+              } else {
+                Token<?> token = fs.getDelegationToken(renewer);
+                Credentials cred = new Credentials();
+                cred.addToken(token.getService(), token);
+                cred.writeTokenStorageFile(tokenFile, conf);
+                System.out.println("Fetched token for " + token.getService()
+                    + " into " + tokenFile);
+              }
+            }
+            return null;
+          }
+        });
   }
   
   static public Credentials getDTfromRemote(String nnAddr, 
@@ -163,7 +220,6 @@ public class DelegationTokenFetcher {
       } else {
         url.append(nnAddr).append(GetDelegationTokenServlet.PATH_SPEC);
       }
-      System.out.println("Retrieving token from: " + url);
       URL remoteURL = new URL(url.toString());
       SecurityUtil.fetchServiceTicket(remoteURL);
       URLConnection connection = remoteURL.openConnection();
@@ -201,7 +257,11 @@ public class DelegationTokenFetcher {
     try {
       URL url = new URL(buf.toString());
       SecurityUtil.fetchServiceTicket(url);
-      URLConnection connection = url.openConnection();
+      HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+      if (connection.getResponseCode() != HttpURLConnection.HTTP_OK) {
+        throw new IOException("Error renewing token: " + 
+            connection.getResponseMessage());
+      }
       in = new BufferedReader(new InputStreamReader
           (connection.getInputStream()));
       long result = Long.parseLong(in.readLine());
@@ -235,7 +295,7 @@ public class DelegationTokenFetcher {
       SecurityUtil.fetchServiceTicket(url);
       HttpURLConnection connection = (HttpURLConnection) url.openConnection();
       if (connection.getResponseCode() != HttpURLConnection.HTTP_OK) {
-        throw new IOException("Error cancelling token:" + 
+        throw new IOException("Error cancelling token: " + 
             connection.getResponseMessage());
       }
     } catch (IOException ie) {
@@ -243,22 +303,4 @@ public class DelegationTokenFetcher {
       throw ie;
     }
   }
-
-
-
-  /**
-   * Utility method to obtain a delegation token over http
-   * @param nnHttpAddr Namenode http addr, such as http://namenode:50070
-   * @param filename Name of file to store token in
-   */
-  static private void getDTfromRemoteIntoFile(String nnAddr, String filename) 
-  throws IOException {
-    Credentials ts = getDTfromRemote(nnAddr, null); 
-
-    DataOutputStream file = new DataOutputStream(new FileOutputStream(filename));
-    ts.writeTokenStorageToStream(file);
-    file.flush();
-    System.out.println("Successfully wrote token of " + file.size() 
-        + " bytes  to " + filename);
-  }
 }

Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/tools/TestDelegationTokenFetcher.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/tools/TestDelegationTokenFetcher.java?rev=1035508&r1=1035507&r2=1035508&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/tools/TestDelegationTokenFetcher.java
(original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/tools/TestDelegationTokenFetcher.java
Tue Nov 16 00:41:17 2010
@@ -17,42 +17,43 @@
  */
 package org.apache.hadoop.tools;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
+import static org.junit.Assert.assertTrue;
 import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
+import java.io.IOException;
 import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Iterator;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileSystemTestHelper;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
 import org.apache.hadoop.hdfs.tools.DelegationTokenFetcher;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
 import org.junit.Before;
 import org.junit.Test;
 
 public class TestDelegationTokenFetcher {
   private DistributedFileSystem dfs;
-  private DataOutputStream out;
-  private UserGroupInformation ugi;
   private Configuration conf;
+  private URI uri;
+  private static final String SERVICE_VALUE = "localhost:2005";
+  private static String tokenFile = "file.dta";
 
   @Before 
-  public void init() {
+  public void init() throws URISyntaxException, IOException {
     dfs = mock(DistributedFileSystem.class);
-    out = mock(DataOutputStream.class);
-    ugi = mock(UserGroupInformation.class);
     conf = new Configuration();
+    uri = new URI("hdfs://" + SERVICE_VALUE);
+    FileSystemTestHelper.addFileSystemForTesting(uri, conf, dfs);
   }
   
   /**
@@ -61,68 +62,42 @@ public class TestDelegationTokenFetcher 
    */
   @Test
   public void expectedTokenIsRetrievedFromDFS() throws Exception {
-    final String LONG_NAME = "TheDoctor@TARDIS";
-    final String SHORT_NAME = "TheDoctor";
-    final String SERVICE_VALUE = "localhost:2005";
-    URI uri = new URI("hdfs://" + SERVICE_VALUE);
-    FileSystem.setDefaultUri(conf, uri);
-    
-    // Mock out the user's long and short names.
-    when(ugi.getUserName()).thenReturn(LONG_NAME);
-    when(ugi.getShortUserName()).thenReturn(SHORT_NAME);
-    
+    final byte[] ident = new DelegationTokenIdentifier(new Text("owner"),
+        new Text("renewer"), new Text("realuser")).getBytes();
+    final byte[] pw = new byte[] { 42 };
+    final Text kind = new Text("MY-KIND");
+    final Text service = new Text(uri.toString());
+
     // Create a token for the fetcher to fetch, wire NN to return it when asked
     // for this particular user.
-    Token<DelegationTokenIdentifier> t = new Token<DelegationTokenIdentifier>();
-    when(dfs.getDelegationToken(eq(LONG_NAME))).thenReturn(t);
-    
-    // Now, actually let the TokenFetcher go fetch the token.
-    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
-    out = new DataOutputStream(baos);
-    new DelegationTokenFetcher(dfs, out, ugi, conf).go();
-    
-    // now read the data back in and verify correct values
-    Credentials ts = new Credentials();
-    DataInputStream dis = 
-      new DataInputStream(new ByteArrayInputStream(baos.toByteArray()));
-    
-    ts.readTokenStorageStream(dis);
-    Token<? extends TokenIdentifier> newToken = ts.getToken(new Text(SHORT_NAME));
-    
-    assertEquals("Should only be one token in storage", ts.numberOfTokens(), 1);
-    assertEquals("Service value should have survived", 
-        "127.0.0.1:2005", newToken.getService().toString());
-  }
+    Token<DelegationTokenIdentifier> t = new Token<DelegationTokenIdentifier>(
+        ident, pw, kind, service);
+    when(dfs.getDelegationToken((String) null)).thenReturn(t);
+    when(dfs.renewDelegationToken(eq(t))).thenReturn(1000L);
+    when(dfs.getUri()).thenReturn(uri);
 
-  private void checkWithNullParam(String s) {
+    FileSystem fileSys = FileSystem.getLocal(conf);
     try {
-      new DelegationTokenFetcher(dfs, out, ugi, conf);
-    } catch (IllegalArgumentException iae) {
-      assertEquals("Expected exception message not received", 
-          s + " cannot be null.", iae.getMessage());
-      return; // received expected exception. We're good.
+      DelegationTokenFetcher.main(new String[] { "-fs", uri.toString(),
+          tokenFile });
+      Path p = new Path(fileSys.getWorkingDirectory(), tokenFile);
+      Credentials creds = Credentials.readTokenStorageFile(p, conf);
+      Iterator<Token<?>> itr = creds.getAllTokens().iterator();
+      // make sure we got back exactly the 1 token we expected
+      assertTrue(itr.hasNext());
+      assertEquals(t, itr.next());
+      assertTrue(!itr.hasNext());
+
+      DelegationTokenFetcher.main(new String[] { "-fs", uri.toString(),
+          "--print", tokenFile });
+      DelegationTokenFetcher.main(new String[] { "-fs", uri.toString(),
+          "--renew", tokenFile });
+      DelegationTokenFetcher.main(new String[] { "-fs", uri.toString(),
+          "--cancel", tokenFile });
+      verify(dfs).renewDelegationToken(eq(t));
+      verify(dfs).cancelDelegationToken(eq(t));
+    } finally {
+      fileSys.delete(new Path(tokenFile), true);
     }
-    fail("null parameter should have failed.");
-  }
-  
-  @Test
-  public void dfsCannotBeNull() {
-    dfs = null;
-    String s = "dfs";
-    checkWithNullParam(s);
-  }
-
-  @Test
-  public void dosCannotBeNull() {
-    out = null;
-    String s = "out";
-    checkWithNullParam(s);
-  }
-  
-  @Test
-  public void ugiCannotBeNull() {
-    ugi = null;
-    String s = "ugi";
-    checkWithNullParam(s);
   }
 }



Mime
View raw message