hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jgho...@apache.org
Subject svn commit: r937914 - in /hadoop/hdfs/trunk: CHANGES.txt src/test/hdfs/org/apache/hadoop/hdfs/TestDFSClientRetries.java
Date Mon, 26 Apr 2010 02:18:13 GMT
Author: jghoman
Date: Mon Apr 26 02:18:12 2010
New Revision: 937914

URL: http://svn.apache.org/viewvc?rev=937914&view=rev
Log:
HDFS-921. Convert TestDFSClientRetries::testNotYetReplicatedErrors to Mockito. jghoman.

Modified:
    hadoop/hdfs/trunk/CHANGES.txt
    hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSClientRetries.java

Modified: hadoop/hdfs/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=937914&r1=937913&r2=937914&view=diff
==============================================================================
--- hadoop/hdfs/trunk/CHANGES.txt (original)
+++ hadoop/hdfs/trunk/CHANGES.txt Mon Apr 26 02:18:12 2010
@@ -153,6 +153,9 @@ Trunk (unreleased changes)
     HDFS-1054. Remove unnecessary sleep after failure in nextBlockOutputStream.
     (Todd Lipcon via jghoman)
 
+    HDFS-921. Convert TestDFSClientRetries::testNotYetReplicatedErrors
+    to Mockito. (jghoman)
+
   OPTIMIZATIONS
 
     HDFS-946. NameNode should not return full path name when lisitng a

Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSClientRetries.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSClientRetries.java?rev=937914&r1=937913&r2=937914&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSClientRetries.java (original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSClientRetries.java Mon Apr
26 02:18:12 2010
@@ -17,39 +17,43 @@
  */
 package org.apache.hadoop.hdfs;
 
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.security.MessageDigest;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
-import java.security.MessageDigest;
+
+import junit.framework.TestCase;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.*;
-import org.apache.hadoop.fs.Options.Rename;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hdfs.DFSInputStream;
-import org.apache.hadoop.hdfs.protocol.*;
-import org.apache.hadoop.hdfs.protocol.FSConstants.UpgradeAction;
-import org.apache.hadoop.hdfs.server.common.*;
-import org.apache.hadoop.hdfs.server.namenode.NotReplicatedYetException;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileChecksum;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.DatanodeID;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
 import org.apache.hadoop.hdfs.server.namenode.NameNode;
-import org.apache.hadoop.io.*;
+import org.apache.hadoop.hdfs.server.namenode.NotReplicatedYetException;
+import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.ipc.RemoteException;
-import org.apache.hadoop.security.AccessControlException;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.SecretManager.InvalidToken;
-import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
-
-import junit.framework.TestCase;
-
-import static org.mockito.Mockito.*;
-import org.mockito.stubbing.Answer;
+import org.mockito.internal.stubbing.answers.ThrowsException;
 import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
 
 /**
  * These tests make sure that DFSClient retries fetching data from DFS
@@ -121,181 +125,50 @@ public class TestDFSClientRetries extend
   }
   
   // more tests related to different failure cases can be added here.
-  
-  class TestNameNode implements ClientProtocol
-  {
-    int num_calls = 0;
-    
-    // The total number of calls that can be made to addBlock
-    // before an exception is thrown
-    int num_calls_allowed; 
-    public final String ADD_BLOCK_EXCEPTION = "Testing exception thrown from"
-                                             + "TestDFSClientRetries::"
-                                             + "TestNameNode::addBlock";
-    public final String RETRY_CONFIG
-          = "dfs.client.block.write.locateFollowingBlock.retries";
-          
-    public TestNameNode(Configuration conf) throws IOException
-    {
-      // +1 because the configuration value is the number of retries and
-      // the first call is not a retry (e.g., 2 retries == 3 total
-      // calls allowed)
-      this.num_calls_allowed = conf.getInt(RETRY_CONFIG, 5) + 1;
-    }
-
-    public long getProtocolVersion(String protocol, 
-                                     long clientVersion)
-    throws IOException
-    {
-      return versionID;
-    }
-
-    public LocatedBlock addBlock(String src, String clientName,
-                                 Block previous) throws IOException {
-
-      return addBlock(src, clientName, previous, null);
-    }
-
-    public LocatedBlock addBlock(String src,
-                                 String clientName,
-                                 Block previous,
-                                 DatanodeInfo[] excludedNode
-                                 ) throws IOException
-    {
-      num_calls++;
-      if (num_calls > num_calls_allowed) { 
-        throw new IOException("addBlock called more times than "
-                              + RETRY_CONFIG
-                              + " allows.");
-      } else {
-          throw new RemoteException(NotReplicatedYetException.class.getName(),
-                                    ADD_BLOCK_EXCEPTION);
-      }
-    }
-    
-    
-    // The following methods are stub methods that are not needed by this mock class
-    
-    public LocatedBlocks  getBlockLocations(String src, long offset, long length) throws
IOException { return null; }
-    
-    public FsServerDefaults getServerDefaults() throws IOException { return null; }
-    
-    public void create(String src, FsPermission masked, String clientName, EnumSetWritable<CreateFlag>
flag, boolean createParent, short replication, long blockSize) throws IOException {}
-    
-    public LocatedBlock append(String src, String clientName) throws IOException { return
null; }
-
-    public boolean setReplication(String src, short replication) throws IOException { return
false; }
-
-    public void setPermission(String src, FsPermission permission) throws IOException {}
-
-    public void setOwner(String src, String username, String groupname) throws IOException
{}
-
-    public void abandonBlock(Block b, String src, String holder) throws IOException {}
-
-    public boolean complete(String src, String clientName, Block last) throws IOException
{ return false; }
-
-    public void reportBadBlocks(LocatedBlock[] blocks) throws IOException {}
-
-    public Token<DelegationTokenIdentifier> getDelegationToken(Text renewer)
-        throws IOException {
-      return null;
-    }
-    
-    public long renewDelegationToken(Token<DelegationTokenIdentifier> token)
-        throws InvalidToken, IOException {
-      return 0;
-    }
-
-    public void cancelDelegationToken(Token<DelegationTokenIdentifier> token)
-        throws IOException {
-    }
-    
-    
-    @Deprecated
-    public boolean rename(String src, String dst) throws IOException { return false; }
-    
-    public void concat(String trg, String[] srcs) throws IOException {  }
 
-    public void rename(String src, String dst, Rename... options) throws IOException { }
-
-    public boolean delete(String src) throws IOException { return false; }
-
-    public boolean delete(String src, boolean recursive) throws IOException { return false;
}
-
-    public boolean mkdirs(String src, FsPermission masked, boolean createParent) throws IOException
{ return false; }
-
-    public DirectoryListing getListing(String src, byte[] startName) throws IOException {
return null; }
-
-    public void renewLease(String clientName) throws IOException {}
-
-    public long[] getStats() throws IOException { return null; }
-
-    public DatanodeInfo[] getDatanodeReport(FSConstants.DatanodeReportType type) throws IOException
{ return null; }
-
-    public long getPreferredBlockSize(String filename) throws IOException { return 0; }
-
-    public boolean setSafeMode(FSConstants.SafeModeAction action) throws IOException { return
false; }
-
-    public void saveNamespace() throws IOException {}
-
-    public boolean restoreFailedStorage(String arg) throws AccessControlException { return
false; }
-
-    public void refreshNodes() throws IOException {}
-
-    public void finalizeUpgrade() throws IOException {}
-
-    public UpgradeStatusReport distributedUpgradeProgress(UpgradeAction action) throws IOException
{ return null; }
-
-    public void metaSave(String filename) throws IOException {}
-
-    public HdfsFileStatus getFileInfo(String src) throws IOException { return null; }
-
-    public HdfsFileStatus getFileLinkInfo(String src) throws IOException { return null; }
-
-    public ContentSummary getContentSummary(String path) throws IOException { return null;
}
-
-    public void setQuota(String path, long namespaceQuota, long diskspaceQuota) throws IOException
{}
-
-    public void fsync(String src, String client) throws IOException {}
-
-    public void setTimes(String src, long mtime, long atime) throws IOException {}
-
-    public FileStatus[] getCorruptFiles()
-      throws AccessControlException, IOException {
-      return null;
-    }
-
-    public void createSymlink(String target, String newpath, 
-        FsPermission dirPerm, boolean createPath) 
-        throws IOException, UnresolvedLinkException {}
-
-    public String getLinkTarget(String src) throws IOException { return null; }
-
-    @Override public LocatedBlock updateBlockForPipeline(Block block, 
-        String clientName) throws IOException { return null; }
-
-    @Override public void updatePipeline(String clientName, Block oldblock,
-        Block newBlock, DatanodeID[] newNodes)
-        throws IOException {}
-  }
-  
+  /**
+   * Verify that client will correctly give up after the specified number
+   * of times trying to add a block
+   */
+  @SuppressWarnings("serial")
   public void testNotYetReplicatedErrors() throws IOException
-  {   
+  { 
+    final String exceptionMsg = "Nope, not replicated yet...";
+    final int maxRetries = 1; // Allow one retry (total of two calls)
     Configuration conf = new HdfsConfiguration();
+   conf.setInt(DFSConfigKeys.DFS_CLIENT_BLOCK_WRITE_LOCATEFOLLOWINGBLOCK_RETRIES_KEY, 
+                maxRetries);
     
-    // allow 1 retry (2 total calls)
-    conf.setInt("dfs.client.block.write.locateFollowingBlock.retries", 1);
-        
-    TestNameNode tnn = new TestNameNode(conf);
-    final DFSClient client = new DFSClient(null, tnn, conf, null);
+    NameNode mockNN = mock(NameNode.class);
+    Answer<Object> answer = new ThrowsException(new IOException()) {
+      int retryCount = 0;
+      
+      @Override
+      public Object answer(InvocationOnMock invocation) 
+                       throws Throwable {
+        retryCount++;
+        System.out.println("addBlock has been called "  + retryCount + " times");
+        if(retryCount > maxRetries + 1) // First call was not a retry
+          throw new IOException("Retried too many times: " + retryCount);
+        else
+          throw new RemoteException(NotReplicatedYetException.class.getName(),
+                                    exceptionMsg);
+      }
+    };
+    when(mockNN.addBlock(anyString(), 
+                         anyString(),
+                         any(Block.class),
+                         any(DatanodeInfo[].class))).thenAnswer(answer);
+
+    final DFSClient client = new DFSClient(null, mockNN, conf, null);
     OutputStream os = client.create("testfile", true);
     os.write(20); // write one random byte
     
     try {
       os.close();
     } catch (Exception e) {
-      assertTrue("Retries are not being stopped correctly",
-           e.getMessage().equals(tnn.ADD_BLOCK_EXCEPTION));
+      assertTrue("Retries are not being stopped correctly: " + e.getMessage(),
+           e.getMessage().equals(exceptionMsg));
     }
   }
 
@@ -422,7 +295,6 @@ public class TestDFSClientRetries extend
     }
   }
   
-  
   /**
    * Test that a DFSClient waits for random time before retry on busy blocks.
    */
@@ -692,3 +564,4 @@ public class TestDFSClientRetries extend
     }
   }
 }
+



Mime
View raw message