hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From te...@apache.org
Subject svn commit: r1230077 - /hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java
Date Wed, 11 Jan 2012 14:58:47 GMT
Author: tedyu
Date: Wed Jan 11 14:58:46 2012
New Revision: 1230077

URL: http://svn.apache.org/viewvc?rev=1230077&view=rev
Log:
HBASE-5163 TestLogRolling#testLogRollOnDatanodeDeath fails sometimes on Jenkins or hadoop
QA ("The directory is already locked.") (N Keywal)

Modified:
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java

Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java?rev=1230077&r1=1230076&r2=1230077&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java
(original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java
Wed Jan 11 14:58:46 2012
@@ -19,51 +19,32 @@
  */
 package org.apache.hadoop.hbase.regionserver.wal;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-import java.io.EOFException;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.commons.logging.impl.Log4JLogger;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.*;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.*;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.JVMClusterUtil;
+import org.apache.hadoop.hbase.util.*;
 import org.apache.hadoop.hdfs.DFSClient;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.hdfs.server.namenode.LeaseManager;
 import org.apache.log4j.Level;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
+import org.junit.*;
 import org.junit.experimental.categories.Category;
 
+import java.io.*;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.*;
+
+import static org.junit.Assert.*;
+
 /**
  * Test log deletion as logs are rolled.
  */
@@ -303,6 +284,7 @@ public class TestLogRolling  {
     return (DatanodeInfo[]) repl;
   }
 
+
   /**
    * Tests that logs are rolled upon detecting datanode death
    * Requires an HDFS jar with HDFS-826 & syncFs() support (HDFS-200)
@@ -313,14 +295,10 @@ public class TestLogRolling  {
    * @throws IllegalArgumentException
     */
   @Test
-  public void testLogRollOnDatanodeDeath() throws IOException,
-      InterruptedException, IllegalArgumentException, IllegalAccessException,
-      InvocationTargetException {
-    assertTrue("This test requires HLog file replication.",
-      fs.getDefaultReplication() > 1);
+  public void testLogRollOnDatanodeDeath() throws Exception {
+    assertTrue("This test requires HLog file replication set to 2.",
+      fs.getDefaultReplication() == 2);
     LOG.info("Replication=" + fs.getDefaultReplication());
-    // When the META table can be opened, the region servers are running
-    new HTable(TEST_UTIL.getConfiguration(), HConstants.META_TABLE_NAME);
 
     this.server = cluster.getRegionServer(0);
     this.log = server.getWAL();
@@ -332,6 +310,7 @@ public class TestLogRolling  {
 
     admin.createTable(desc);
     HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
+    assertTrue(table.isAutoFlush());
 
     server = TEST_UTIL.getRSForFirstRegionInTable(Bytes.toBytes(tableName));
     this.log = server.getWAL();
@@ -342,27 +321,35 @@ public class TestLogRolling  {
         .isAppendSupported(TEST_UTIL.getConfiguration()));
 
     // add up the datanode count, to ensure proper replication when we kill 1
+    // This function is synchronous; when it returns, the dfs cluster is active
+    // We start 3 servers and then stop 2 to avoid a directory naming conflict
+    //  when we stop/start a namenode later, as mentioned in HBASE-5163
+    List<DataNode> existingNodes = dfsCluster.getDataNodes();
     dfsCluster
-        .startDataNodes(TEST_UTIL.getConfiguration(), 1, true, null, null);
-    dfsCluster.waitActive();
-    assertTrue(dfsCluster.getDataNodes().size() >= fs.getDefaultReplication() + 1);
+        .startDataNodes(TEST_UTIL.getConfiguration(), 3, true, null, null);
+    for (DataNode dn: existingNodes){
+      dfsCluster.stopDataNode( dn.dnRegistration.getName() );
+    }
 
-    writeData(table, 2);
+    assertTrue(
+      dfsCluster.getDataNodes().size() >= fs.getDefaultReplication() + 1);
 
-    table.setAutoFlush(true);
+    writeData(table, 2);
 
     long curTime = System.currentTimeMillis();
     long oldFilenum = log.getFilenum();
     assertTrue("Log should have a timestamp older than now",
         curTime > oldFilenum && oldFilenum != -1);
 
-    assertTrue("The log shouldn't have rolled yet", oldFilenum == log.getFilenum());
-    DatanodeInfo[] pipeline = getPipeline(log);
+    assertTrue("The log shouldn't have rolled yet",
+      oldFilenum == log.getFilenum());
+    final DatanodeInfo[] pipeline = getPipeline(log);
     assertTrue(pipeline.length == fs.getDefaultReplication());
 
     // kill a datanode in the pipeline to force a log roll on the next sync()
+    // This function is synchronous, when it returns the node is killed.
     assertTrue(dfsCluster.stopDataNode(pipeline[0].getName()) != null);
-    Thread.sleep(10000);
+
     // this write should succeed, but trigger a log roll
     writeData(table, 2);
     long newFilenum = log.getFilenum();
@@ -372,23 +359,26 @@ public class TestLogRolling  {
 
     // write some more log data (this should use a new hdfs_out)
     writeData(table, 3);
-    assertTrue("The log should not roll again.", log.getFilenum() == newFilenum);
+    assertTrue("The log should not roll again.",
+      log.getFilenum() == newFilenum);
     // kill another datanode in the pipeline, so the replicas will be lower than
     // the configured value 2.
     assertTrue(dfsCluster.stopDataNode(pipeline[1].getName()) != null);
-    Thread.sleep(10000);
+
     batchWriteAndWait(table, 3, false, 10000);
     assertTrue("LowReplication Roller should've been disabled",
         !log.isLowReplicationRollEnabled());
+
     dfsCluster
         .startDataNodes(TEST_UTIL.getConfiguration(), 1, true, null, null);
-    dfsCluster.waitActive();
+
     // Force roll writer. The new log file will have the default replications,
     // and the LowReplication Roller will be enabled.
     log.rollWriter(true);
     batchWriteAndWait(table, 13, true, 10000);
-    assertTrue("New log file should have the default replication",
-        log.getLogReplication() == fs.getDefaultReplication());
+    assertTrue("New log file should have the default replication instead of " +
+      log.getLogReplication(),
+      log.getLogReplication() == fs.getDefaultReplication());
     assertTrue("LowReplication Roller should've been enabled",
         log.isLowReplicationRollEnabled());
   }



Mime
View raw message