hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kih...@apache.org
Subject svn commit: r1482122 - in /hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs: CHANGES.txt src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java
Date Mon, 13 May 2013 22:06:56 GMT
Author: kihwal
Date: Mon May 13 22:06:55 2013
New Revision: 1482122

URL: http://svn.apache.org/r1482122
Log:
svn merge -c 1482121. Merging from trunk to branch-2 to fix HDFS-4805.

Modified:
    hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
    hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
    hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java

Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1482122&r1=1482121&r2=1482122&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Mon May 13
22:06:55 2013
@@ -2329,6 +2329,9 @@ Release 0.23.8 - UNRELEASED
     HDFS-4699. TestPipelinesFailover#testPipelineRecoveryStress fails
     sporadically (Chris Nauroth via kihwal)
 
+    HDFS-4805. Webhdfs client is fragile to token renewal errors 
+    (daryn via kihwal)
+
 Release 0.23.7 - UNRELEASED
 
   INCOMPATIBLE CHANGES

Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java?rev=1482122&r1=1482121&r2=1482122&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
Mon May 13 22:06:55 2013
@@ -133,6 +133,8 @@ public class WebHdfsFileSystem extends F
       = new WebHdfsDelegationTokenSelector();
 
   private DelegationTokenRenewer dtRenewer = null;
+  @VisibleForTesting
+  DelegationTokenRenewer.RenewAction<?> action;
 
   @VisibleForTesting
   protected synchronized void addRenewAction(final WebHdfsFileSystem webhdfs) {
@@ -140,7 +142,7 @@ public class WebHdfsFileSystem extends F
       dtRenewer = DelegationTokenRenewer.getInstance();
     }
 
-    dtRenewer.addRenewAction(webhdfs);
+    action = dtRenewer.addRenewAction(webhdfs);
   }
 
   /** Is WebHDFS enabled in conf? */
@@ -208,7 +210,8 @@ public class WebHdfsFileSystem extends F
   }
 
   protected synchronized Token<?> getDelegationToken() throws IOException {
-    if (!hasInitedToken) {
+    // we haven't inited yet, or we used to have a token but it expired
+    if (!hasInitedToken || (action != null && !action.isValid())) {
       //since we don't already have a token, go get one
       Token<?> token = getDelegationToken(null);
       // security might be disabled

Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java?rev=1482122&r1=1482121&r2=1482122&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java
Mon May 13 22:06:55 2013
@@ -27,6 +27,8 @@ import java.io.IOException;
 import java.net.URI;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.DelegationTokenRenewer;
+import org.apache.hadoop.fs.DelegationTokenRenewer.RenewAction;
 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
 import org.apache.hadoop.hdfs.web.resources.DeleteOpParam;
 import org.apache.hadoop.hdfs.web.resources.GetOpParam;
@@ -199,4 +201,50 @@ public class TestWebHdfsTokens {
       assertFalse(op.getRequireAuth());
     }
   }
+  
+  @Test
+  public void testGetTokenAfterFailure() throws Exception {
+    Configuration conf = mock(Configuration.class);
+    Token<?> token1 = mock(Token.class);
+    Token<?> token2 = mock(Token.class);
+    long renewCycle = 1000;
+    
+    DelegationTokenRenewer.renewCycle = renewCycle;
+    WebHdfsFileSystem fs = spy(new WebHdfsFileSystem());
+    doReturn(conf).when(fs).getConf();
+    doReturn(token1).doReturn(token2).when(fs).getDelegationToken(null);
+    // cause token renewer to abandon the token
+    doThrow(new IOException("renew failed")).when(token1).renew(conf);
+    doThrow(new IOException("get failed")).when(fs).addDelegationTokens(null, null);
+
+    // trigger token acquisition
+    Token<?> token = fs.getDelegationToken();
+    RenewAction<?> action = fs.action; 
+    assertSame(token1, token);
+    assertTrue(action.isValid());
+
+    // fetch again and make sure it's the same as before
+    token = fs.getDelegationToken();
+    assertSame(token1, token);
+    assertSame(action, fs.action);
+    assertTrue(fs.action.isValid());
+    
+    // upon renewal, token will go bad based on above stubbing
+    Thread.sleep(renewCycle);
+    assertSame(action, fs.action);
+    assertFalse(fs.action.isValid());
+    
+    // now that token is invalid, should get a new one
+    token = fs.getDelegationToken();
+    assertSame(token2, token);
+    assertNotSame(action, fs.action);
+    assertTrue(fs.action.isValid());
+    action = fs.action;
+    
+    // should get same one again
+    token = fs.getDelegationToken();
+    assertSame(token2, token);
+    assertSame(action, fs.action);
+    assertTrue(fs.action.isValid());
+  }
 }



Mime
View raw message