Return-Path: X-Original-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id D8115CD8F for ; Tue, 12 Jun 2012 05:27:42 +0000 (UTC) Received: (qmail 69625 invoked by uid 500); 12 Jun 2012 05:27:41 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 69414 invoked by uid 500); 12 Jun 2012 05:27:40 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 69368 invoked by uid 99); 12 Jun 2012 05:27:39 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 12 Jun 2012 05:27:39 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 12 Jun 2012 05:27:37 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id A4C942388AF0; Tue, 12 Jun 2012 05:27:17 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1349124 - in /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs: ./ src/main/java/org/apache/hadoop/hdfs/ src/test/java/org/apache/hadoop/hdfs/ Date: Tue, 12 Jun 2012 05:27:17 -0000 To: hdfs-commits@hadoop.apache.org From: szetszwo@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120612052717.A4C942388AF0@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: szetszwo Date: Tue Jun 12 05:27:15 2012 New Revision: 1349124 URL: http://svn.apache.org/viewvc?rev=1349124&view=rev Log: HDFS-3504. Support configurable retry policy in DFSClient for RPC connections and RPC calls, and add MultipleLinearRandomRetry, a new retry policy. Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1349124&r1=1349123&r2=1349124&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Tue Jun 12 05:27:15 2012 @@ -222,6 +222,10 @@ Branch-2 ( Unreleased changes ) HDFS-3520. Add transfer rate logging to TransferFsImage. (eli) + HDFS-3504. Support configurable retry policy in DFSClient for RPC + connections and RPC calls, and add MultipleLinearRandomRetry, a new retry + policy. (szetszwo) + OPTIMIZATIONS HDFS-2982. Startup performance suffers when there are many edit log Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1349124&r1=1349123&r2=1349124&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java Tue Jun 12 05:27:15 2012 @@ -38,6 +38,10 @@ public class DFSConfigKeys extends Commo public static final int DFS_STREAM_BUFFER_SIZE_DEFAULT = 4096; public static final String DFS_BYTES_PER_CHECKSUM_KEY = "dfs.bytes-per-checksum"; public static final int DFS_BYTES_PER_CHECKSUM_DEFAULT = 512; + public static final String DFS_CLIENT_RETRY_POLICY_ENABLED_KEY = "dfs.client.retry.policy.enabled"; + public static final boolean DFS_CLIENT_RETRY_POLICY_ENABLED_DEFAULT = false; + public static final String DFS_CLIENT_RETRY_POLICY_SPEC_KEY = "dfs.client.retry.policy.spec"; + public static final String DFS_CLIENT_RETRY_POLICY_SPEC_DEFAULT = "10000,6,60000,10"; //t1,n1,t2,n2,... public static final String DFS_CHECKSUM_TYPE_KEY = "dfs.checksum.type"; public static final String DFS_CHECKSUM_TYPE_DEFAULT = "CRC32C"; public static final String DFS_CLIENT_WRITE_PACKET_SIZE_KEY = "dfs.client-write-packet-size"; Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java?rev=1349124&r1=1349123&r2=1349124&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java Tue Jun 12 05:27:15 2012 @@ -47,10 +47,12 @@ import org.apache.hadoop.hdfs.protocolPB import org.apache.hadoop.hdfs.protocolPB.RefreshUserMappingsProtocolClientSideTranslatorPB; import org.apache.hadoop.hdfs.protocolPB.RefreshUserMappingsProtocolPB; import org.apache.hadoop.hdfs.server.namenode.NameNode; +import org.apache.hadoop.hdfs.server.namenode.SafeModeException; import org.apache.hadoop.hdfs.server.protocol.JournalProtocol; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.retry.DefaultFailoverProxyProvider; import org.apache.hadoop.io.retry.FailoverProxyProvider; import org.apache.hadoop.io.retry.RetryPolicies; import org.apache.hadoop.io.retry.RetryPolicy; @@ -66,6 +68,7 @@ import org.apache.hadoop.security.author import org.apache.hadoop.tools.GetUserMappingsProtocol; import com.google.common.base.Preconditions; +import com.google.protobuf.ServiceException; /** * Create proxy objects to communicate with a remote NN. All remote access to an @@ -240,12 +243,106 @@ public class NameNodeProxies { return new NamenodeProtocolTranslatorPB(proxy); } + /** + * Return the default retry policy used in RPC. + * + * If dfs.client.retry.policy.enabled == false, use TRY_ONCE_THEN_FAIL. + * + * Otherwise, first unwrap ServiceException if possible, and then + * (1) use multipleLinearRandomRetry for + * - SafeModeException, or + * - IOException other than RemoteException, or + * - ServiceException; and + * (2) use TRY_ONCE_THEN_FAIL for + * - non-SafeMode RemoteException, or + * - non-IOException. + * + * Note that dfs.client.retry.max < 0 is not allowed. + */ + private static RetryPolicy getDefaultRpcRetryPolicy(Configuration conf) { + final RetryPolicy multipleLinearRandomRetry = getMultipleLinearRandomRetry(conf); + if (LOG.isDebugEnabled()) { + LOG.debug("multipleLinearRandomRetry = " + multipleLinearRandomRetry); + } + if (multipleLinearRandomRetry == null) { + //no retry + return RetryPolicies.TRY_ONCE_THEN_FAIL; + } else { + return new RetryPolicy() { + @Override + public RetryAction shouldRetry(Exception e, int retries, int failovers, + boolean isMethodIdempotent) throws Exception { + if (e instanceof ServiceException) { + //unwrap ServiceException + final Throwable cause = e.getCause(); + if (cause != null && cause instanceof Exception) { + e = (Exception)cause; + } + } + + //see (1) and (2) in the javadoc of this method. + final RetryPolicy p; + if (e instanceof RemoteException) { + final RemoteException re = (RemoteException)e; + p = SafeModeException.class.getName().equals(re.getClassName())? + multipleLinearRandomRetry: RetryPolicies.TRY_ONCE_THEN_FAIL; + } else if (e instanceof IOException || e instanceof ServiceException) { + p = multipleLinearRandomRetry; + } else { //non-IOException + p = RetryPolicies.TRY_ONCE_THEN_FAIL; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("RETRY " + retries + ") policy=" + + p.getClass().getSimpleName() + ", exception=" + e); + } + LOG.info("RETRY " + retries + ") policy=" + + p.getClass().getSimpleName() + ", exception=" + e); + return p.shouldRetry(e, retries, failovers, isMethodIdempotent); + } + }; + } + } + + /** + * Return the MultipleLinearRandomRetry policy specified in the conf, + * or null if the feature is disabled. + * If the policy is specified in the conf but the policy cannot be parsed, + * the default policy is returned. + * + * Conf property: N pairs of sleep-time and number-of-retries + * dfs.client.retry.policy = "s1,n1,s2,n2,..." + */ + private static RetryPolicy getMultipleLinearRandomRetry(Configuration conf) { + final boolean enabled = conf.getBoolean( + DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_ENABLED_KEY, + DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_ENABLED_DEFAULT); + if (!enabled) { + return null; + } + + final String policy = conf.get( + DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_SPEC_KEY, + DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_SPEC_DEFAULT); + + final RetryPolicy r = RetryPolicies.MultipleLinearRandomRetry.parseCommaSeparatedString(policy); + return r != null? r: RetryPolicies.MultipleLinearRandomRetry.parseCommaSeparatedString( + DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_SPEC_DEFAULT); + } + private static ClientProtocol createNNProxyWithClientProtocol( InetSocketAddress address, Configuration conf, UserGroupInformation ugi, boolean withRetries) throws IOException { - ClientNamenodeProtocolPB proxy = (ClientNamenodeProtocolPB) NameNodeProxies - .createNameNodeProxy(address, conf, ugi, ClientNamenodeProtocolPB.class, 0); + RPC.setProtocolEngine(conf, ClientNamenodeProtocolPB.class, ProtobufRpcEngine.class); + + final RetryPolicy defaultPolicy = getDefaultRpcRetryPolicy(conf); + final long version = RPC.getProtocolVersion(ClientNamenodeProtocolPB.class); + ClientNamenodeProtocolPB proxy = RPC.getProtocolProxy( + ClientNamenodeProtocolPB.class, version, address, ugi, conf, + NetUtils.getDefaultSocketFactory(conf), 0, defaultPolicy).getProxy(); + if (withRetries) { // create the proxy with retries + RetryPolicy createPolicy = RetryPolicies .retryUpToMaximumCountWithFixedSleep(5, HdfsConstants.LEASE_SOFTLIMIT_PERIOD, TimeUnit.MILLISECONDS); @@ -258,17 +355,21 @@ public class NameNodeProxies { Map, RetryPolicy> exceptionToPolicyMap = new HashMap, RetryPolicy>(); exceptionToPolicyMap.put(RemoteException.class, RetryPolicies - .retryByRemoteException(RetryPolicies.TRY_ONCE_THEN_FAIL, + .retryByRemoteException(defaultPolicy, remoteExceptionToPolicyMap)); RetryPolicy methodPolicy = RetryPolicies.retryByException( - RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap); + defaultPolicy, exceptionToPolicyMap); Map methodNameToPolicyMap = new HashMap(); methodNameToPolicyMap.put("create", methodPolicy); - proxy = (ClientNamenodeProtocolPB) RetryProxy - .create(ClientNamenodeProtocolPB.class, proxy, methodNameToPolicyMap); + proxy = (ClientNamenodeProtocolPB) RetryProxy.create( + ClientNamenodeProtocolPB.class, + new DefaultFailoverProxyProvider( + ClientNamenodeProtocolPB.class, proxy), + methodNameToPolicyMap, + defaultPolicy); } return new ClientNamenodeProtocolTranslatorPB(proxy); } Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java?rev=1349124&r1=1349123&r2=1349124&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java Tue Jun 12 05:27:15 2012 @@ -25,8 +25,6 @@ import static org.apache.hadoop.hdfs.DFS import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HOST_NAME_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HTTP_ADDRESS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_IPC_ADDRESS_KEY; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICES; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICE_ID; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY; @@ -39,6 +37,8 @@ import static org.apache.hadoop.hdfs.DFS import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICES; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICE_ID; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY; import static org.apache.hadoop.hdfs.server.common.Util.fileAsURI; @@ -66,12 +66,9 @@ import org.apache.hadoop.conf.Configurat import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.ha.HAServiceProtocol; +import org.apache.hadoop.ha.HAServiceProtocol.RequestSource; import org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo; -import org.apache.hadoop.ha.HAServiceProtocolHelper; import org.apache.hadoop.ha.ServiceFailedException; -import org.apache.hadoop.ha.HAServiceProtocol.RequestSource; -import org.apache.hadoop.ha.protocolPB.HAServiceProtocolClientSideTranslatorPB; import org.apache.hadoop.hdfs.MiniDFSNNTopology.NNConf; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; @@ -1401,7 +1398,6 @@ public class MiniDFSCluster { waitClusterUp(); LOG.info("Restarted the namenode"); waitActive(); - LOG.info("Cluster is active"); } } @@ -1777,6 +1773,7 @@ public class MiniDFSCluster { } } } + LOG.info("Cluster is active"); } private synchronized boolean shouldWait(DatanodeInfo[] dnInfo, Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java?rev=1349124&r1=1349123&r2=1349124&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java Tue Jun 12 05:27:15 2012 @@ -25,46 +25,53 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; -import java.net.SocketTimeoutException; -import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.io.LongWritable; +import java.io.FileNotFoundException; import java.io.IOException; -import java.net.InetSocketAddress; import java.io.InputStream; import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.net.SocketTimeoutException; import java.security.MessageDigest; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.concurrent.TimeUnit; import junit.framework.TestCase; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileChecksum; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnresolvedLinkException; -import org.apache.hadoop.hdfs.DFSConfigKeys; -import org.apache.hadoop.hdfs.protocol.DatanodeID; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol; +import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; +import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.NotReplicatedYetException; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; -import org.apache.hadoop.ipc.RemoteException; +import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.retry.RetryPolicies.MultipleLinearRandomRetry; import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.ipc.Server; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.test.GenericTestUtils; +import org.apache.log4j.Level; import org.mockito.Mockito; import org.mockito.internal.stubbing.answers.ThrowsException; import org.mockito.invocation.InvocationOnMock; @@ -341,7 +348,7 @@ public class TestDFSClientRetries extend // We shouldn't have gained an extra block by the RPC. assertEquals(blockCount, blockCount2); - return (LocatedBlock) ret2; + return ret2; } }).when(spyNN).addBlock(Mockito.anyString(), Mockito.anyString(), Mockito.any(), Mockito.any()); @@ -798,5 +805,149 @@ public class TestDFSClientRetries extend cluster.shutdown(); } } -} + /** Test client retry with namenode restarting. */ + public void testNamenodeRestart() throws Exception { + ((Log4JLogger)DFSClient.LOG).getLogger().setLevel(Level.ALL); + + final List exceptions = new ArrayList(); + + final Path dir = new Path("/testNamenodeRestart"); + + final Configuration conf = new Configuration(); + conf.setBoolean(DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_ENABLED_KEY, true); + + final short numDatanodes = 3; + final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(numDatanodes) + .build(); + try { + cluster.waitActive(); + + //create a file + final DistributedFileSystem dfs = cluster.getFileSystem(); + final long length = 1L << 20; + final Path file1 = new Path(dir, "foo"); + DFSTestUtil.createFile(dfs, file1, length, numDatanodes, 20120406L); + + //get file status + final FileStatus s1 = dfs.getFileStatus(file1); + assertEquals(length, s1.getLen()); + + //shutdown namenode + cluster.shutdownNameNode(0); + + //namenode is down, create another file in a thread + final Path file3 = new Path(dir, "file"); + final Thread thread = new Thread(new Runnable() { + @Override + public void run() { + try { + //it should retry till namenode is up. + final FileSystem fs = AppendTestUtil.createHdfsWithDifferentUsername(conf); + DFSTestUtil.createFile(fs, file3, length, numDatanodes, 20120406L); + } catch (Exception e) { + exceptions.add(e); + } + } + }); + thread.start(); + + //restart namenode in a new thread + new Thread(new Runnable() { + @Override + public void run() { + try { + //sleep, restart, and then wait active + TimeUnit.SECONDS.sleep(30); + cluster.restartNameNode(0, false); + cluster.waitActive(); + } catch (Exception e) { + exceptions.add(e); + } + } + }).start(); + + //namenode is down, it should retry until namenode is up again. + final FileStatus s2 = dfs.getFileStatus(file1); + assertEquals(s1, s2); + + //check file1 and file3 + thread.join(); + assertEquals(dfs.getFileChecksum(file1), dfs.getFileChecksum(file3)); + + //enter safe mode + dfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); + + //leave safe mode in a new thread + new Thread(new Runnable() { + @Override + public void run() { + try { + //sleep and then leave safe mode + TimeUnit.SECONDS.sleep(30); + dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); + } catch (Exception e) { + exceptions.add(e); + } + } + }).start(); + + //namenode is in safe mode, create should retry until it leaves safe mode. + final Path file2 = new Path(dir, "bar"); + DFSTestUtil.createFile(dfs, file2, length, numDatanodes, 20120406L); + assertEquals(dfs.getFileChecksum(file1), dfs.getFileChecksum(file2)); + + //make sure it won't retry on exceptions like FileNotFoundException + final Path nonExisting = new Path(dir, "nonExisting"); + LOG.info("setPermission: " + nonExisting); + try { + dfs.setPermission(nonExisting, new FsPermission((short)0)); + fail(); + } catch(FileNotFoundException fnfe) { + LOG.info("GOOD!", fnfe); + } + + if (!exceptions.isEmpty()) { + LOG.error("There are " + exceptions.size() + " exception(s):"); + for(int i = 0; i < exceptions.size(); i++) { + LOG.error("Exception " + i, exceptions.get(i)); + } + fail(); + } + } finally { + cluster.shutdown(); + } + } + + public void testMultipleLinearRandomRetry() { + parseMultipleLinearRandomRetry(null, ""); + parseMultipleLinearRandomRetry(null, "11"); + parseMultipleLinearRandomRetry(null, "11,22,33"); + parseMultipleLinearRandomRetry(null, "11,22,33,44,55"); + parseMultipleLinearRandomRetry(null, "AA"); + parseMultipleLinearRandomRetry(null, "11,AA"); + parseMultipleLinearRandomRetry(null, "11,22,33,FF"); + parseMultipleLinearRandomRetry(null, "11,-22"); + parseMultipleLinearRandomRetry(null, "-11,22"); + + parseMultipleLinearRandomRetry("[22x11ms]", + "11,22"); + parseMultipleLinearRandomRetry("[22x11ms, 44x33ms]", + "11,22,33,44"); + parseMultipleLinearRandomRetry("[22x11ms, 44x33ms, 66x55ms]", + "11,22,33,44,55,66"); + parseMultipleLinearRandomRetry("[22x11ms, 44x33ms, 66x55ms]", + " 11, 22, 33, 44, 55, 66 "); + } + + static void parseMultipleLinearRandomRetry(String expected, String s) { + final MultipleLinearRandomRetry r = MultipleLinearRandomRetry.parseCommaSeparatedString(s); + LOG.info("input=" + s + ", parsed=" + r + ", expected=" + expected); + if (r == null) { + assertEquals(expected, null); + } else { + assertEquals("MultipleLinearRandomRetry" + expected, r.toString()); + } + } +}