hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mbau...@apache.org
Subject svn commit: r1308651 [2/2] - in /hbase/branches/0.89-fb/src: main/java/org/apache/hadoop/hbase/ main/java/org/apache/hadoop/hbase/master/ main/java/org/apache/hadoop/hbase/regionserver/ main/java/org/apache/hadoop/hbase/regionserver/wal/ main/java/org/...
Date Tue, 03 Apr 2012 02:04:27 GMT
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestLogSplitOnMasterFailover.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestLogSplitOnMasterFailover.java?rev=1308651&r1=1308650&r2=1308651&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestLogSplitOnMasterFailover.java
(original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestLogSplitOnMasterFailover.java
Tue Apr  3 02:04:26 2012
@@ -65,7 +65,7 @@ public class TestLogSplitOnMasterFailove
 
   private static final int NUM_MASTERS = 2;
   private static final int NUM_RS = 2;
-  private static final int NUM_ROWS = 8000;
+  private static final int NUM_ROWS = 6000;
   private static final int COLS_PER_ROW = 30;
 
   private static final byte[] TABLE_BYTES = Bytes.toBytes("myTable");
@@ -73,6 +73,8 @@ public class TestLogSplitOnMasterFailove
 
   private static Compression.Algorithm COMPRESSION = Compression.Algorithm.GZ;
 
+  private Set<String> logsSplitByNewMaster = new HashSet<String>();
+
   /**
    * A worker that inserts data into HBase on a separate thread. This is the
    * reusable base class, which is subclassed for use in this particular test.
@@ -88,12 +90,17 @@ public class TestLogSplitOnMasterFailove
 
     private Semaphore halfRowsLoaded = new Semaphore(0);
     private Semaphore dataLoadVerifyFinished = new Semaphore(0);
+    private Semaphore newMasterAssignedRegions = new Semaphore(0);
 
     private final Configuration conf;
+    private final HBaseTestingUtility testUtil;
+
     private volatile Thread myThread;
+    private volatile int numUserRegions;
 
-    public DataLoader(Configuration conf) {
+    public DataLoader(Configuration conf, HBaseTestingUtility testUtil) {
       this.conf = conf;
+      this.testUtil = testUtil;
     }
 
     @Override
@@ -101,7 +108,7 @@ public class TestLogSplitOnMasterFailove
       myThread = Thread.currentThread();
       myThread.setName(getClass().getSimpleName());
       try {
-        HBaseTestingUtility.createPreSplitLoadTestTable(conf,
+        numUserRegions = HBaseTestingUtility.createPreSplitLoadTestTable(conf,
             TABLE_BYTES, CF_BYTES, COMPRESSION, DataBlockEncoding.NONE);
         t = new HTable(conf, TABLE_BYTES);
 
@@ -122,7 +129,7 @@ public class TestLogSplitOnMasterFailove
       }
     }
 
-    private void loadData() throws IOException {
+    private void loadData() throws IOException, InterruptedException {
       Random rand = new Random(190879817L);
       int bytesInserted = 0;
       for (int i = 0; i < NUM_ROWS; ++i) {
@@ -153,6 +160,8 @@ public class TestLogSplitOnMasterFailove
           LOG.info("Loaded half of the rows (" + rowsLoaded
               + "), waking up main thread");
           halfRowsLoaded.release();
+          newMasterAssignedRegions.acquire();
+          LOG.info("All regions assigned, proceeding with load test");
         }
       }
       LOG.info("Approximate number of bytes inserted: " + bytesInserted);
@@ -193,6 +202,11 @@ public class TestLogSplitOnMasterFailove
       LOG.debug("Waiting until half of the rows are loaded");
       halfRowsLoaded.acquire();
     }
+
+    public void notifyThatNewMasterAssignedRegions() {
+      newMasterAssignedRegions.release();
+    }
+
     public void requestShutdown() {
       shutdownRequested = true;
     }
@@ -209,12 +223,21 @@ public class TestLogSplitOnMasterFailove
     public void join() throws InterruptedException {
       myThread.join();
     }
+
+    public void waitUntilRegionsAssigned() throws IOException{
+      header("Waiting until all " + numUserRegions
+          + " regions are online with new master");
+      testUtil.waitUntilAllRegionsAssigned(numUserRegions);
+      // Tell the load test to proceed.
+      notifyThatNewMasterAssignedRegions();
+    }
   }
 
   @Test(timeout=180000)
   public void testWithRegularLogSplitting() throws Exception {
     ZooKeeperWrapper.setNamespaceForTesting();
     conf.setBoolean(HConstants.DISTRIBUTED_LOG_SPLITTING_KEY, false);
+    conf.setInt(HConstants.ZOOKEEPER_SESSION_TIMEOUT, 30000);
     runTest();
   }
 
@@ -235,8 +258,7 @@ public class TestLogSplitOnMasterFailove
     List<HMaster> masters = miniCluster().getMasters();
 
     header("Starting data loader");
-    DataLoader dataLoader =
-        new DataLoader(conf);
+    DataLoader dataLoader = new DataLoader(conf, TEST_UTIL);
     Thread inserterThread = new Thread(dataLoader);
     inserterThread.start();
     dataLoader.waitUntilHalfRowsLoaded();
@@ -253,9 +275,16 @@ public class TestLogSplitOnMasterFailove
           miniCluster().getRegionServer(i).getServerInfo().getServerName());
     }
     rsToKill.kill();
-    // Wait until the regionserver actually goes down.
-    while (miniCluster().getLiveRegionServerThreads().size() == NUM_RS) {
-      Threads.sleep(HConstants.SOCKET_RETRY_WAIT_MS);
+
+    // Wait until the regionserver actually goes down. Furthermore, to make
+    // things more interesting, wait until the active master starts splitting
+    // the logs before we kill it. We have to do this in a tight polling loop
+    // because with distributed log splitting the active master might actually
+    // be able to split the dead regionserver's logs really quickly.
+    HMaster activeMaster = miniCluster().getMaster(activeIndex);
+    while (activeMaster.getNumDeadServerLogSplitRequests() == 0 ||
+        miniCluster().getLiveRegionServerThreads().size() == NUM_RS) {
+      Threads.sleepWithoutInterrupt(10);
     }
 
     // Check that we have some logs.
@@ -283,6 +312,8 @@ public class TestLogSplitOnMasterFailove
     HMaster master = masters.get(0);
     assertTrue(master.isActiveMaster());
 
+    dataLoader.waitUntilRegionsAssigned();
+
     dataLoader.waitUntilFinishedOrFailed();
     dataLoader.join();
     dataLoader.assertSuccess();
@@ -291,16 +322,22 @@ public class TestLogSplitOnMasterFailove
     List<String> logDirsSplitAtStartup = master.getLogDirsSplitOnStartup();
     LOG.info("Log dirs split at startup: " + logDirsSplitAtStartup);
 
-    Set<String> logsSplit = new HashSet<String>();
-    logsSplit.addAll(logDirsSplitAtStartup);
-    String logDirToBeSplit = killedRsName + "-splitting";
+    logsSplitByNewMaster.addAll(logDirsSplitAtStartup);
+    String logDirToBeSplit = killedRsName + HConstants.HLOG_SPLITTING_EXT;
     assertTrue("Log directory " + logDirToBeSplit + " was not split " +
         "on startup. Logs split: " + logDirsSplitAtStartup,
-        logsSplit.contains(logDirToBeSplit));
-    for (String logNotToSplit : otherRsNames) {
-      assertFalse("Log directory " + logNotToSplit
-          + " should not have been split", logsSplit.contains(logNotToSplit));
+        logWasSplit(logDirToBeSplit));
+    for (String logDirNotToSplit : otherRsNames) {
+      assertFalse("Log directory " + logDirNotToSplit
+          + " should not have been split: " + logDirsSplitAtStartup,
+          logWasSplit(logDirNotToSplit));
     }
   }
 
+  private boolean logWasSplit(String rsName) {
+    return logsSplitByNewMaster.contains(rsName)
+        || logsSplitByNewMaster.contains(rsName
+            + HConstants.HLOG_SPLITTING_EXT);
+  }
+
 }

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java?rev=1308651&r1=1308650&r2=1308651&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
(original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
Tue Apr  3 02:04:26 2012
@@ -71,6 +71,8 @@ public class TestMasterFailover extends 
 
     killActiveMasterAndWaitToStop();
     assertEquals(1, masters.size());
+
+    waitForActiveMasterAndVerify();
   }
 
 }

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestOldLogsCleaner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestOldLogsCleaner.java?rev=1308651&r1=1308650&r2=1308651&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestOldLogsCleaner.java
(original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestOldLogsCleaner.java
Tue Apr  3 02:04:26 2012
@@ -31,6 +31,7 @@ import org.junit.Test;
 
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.StoppableImpl;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.conf.Configuration;
@@ -82,7 +83,7 @@ public class TestOldLogsCleaner {
     String fakeMachineName = URLEncoder.encode("regionserver:60020", "UTF8");
 
     FileSystem fs = FileSystem.get(c);
-    AtomicBoolean stop = new AtomicBoolean(false);
+    StoppableImpl stop = new StoppableImpl();
     OldLogsCleaner cleaner = new OldLogsCleaner(1000, stop,c, fs, oldLogDir);
 
     // Create 2 invalid files, 1 "recent" file, 1 very new file and 30 old files

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestRegionServerOperationQueue.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestRegionServerOperationQueue.java?rev=1308651&r1=1308650&r2=1308651&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestRegionServerOperationQueue.java
(original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestRegionServerOperationQueue.java
Tue Apr  3 02:04:26 2012
@@ -20,10 +20,8 @@
 package org.apache.hadoop.hbase.master;
 import static org.junit.Assert.*;
 
-import java.util.concurrent.atomic.AtomicBoolean;
-
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.StopStatus;
 import org.apache.hadoop.hbase.master.RegionServerOperationQueue.ProcessingResultCode;
 import org.junit.After;
 import org.junit.Before;
@@ -37,13 +35,17 @@ import org.junit.Test;
 public class TestRegionServerOperationQueue {
   private RegionServerOperationQueue queue;
   private Configuration conf;
-  private AtomicBoolean closed;
+  private StopStatus stopStatus = new StopStatus() {
+    @Override
+    public boolean isStopped() {
+      return false;
+    }
+  };
 
   @Before
   public void setUp() throws Exception {
-    this.closed = new AtomicBoolean(false);
     this.conf = new Configuration();
-    this.queue = new RegionServerOperationQueue(this.conf, this.closed);
+    this.queue = new RegionServerOperationQueue(this.conf, stopStatus);
   }
 
   @After

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestRegionStateOnMasterFailure.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestRegionStateOnMasterFailure.java?rev=1308651&r1=1308650&r2=1308651&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestRegionStateOnMasterFailure.java
(original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestRegionStateOnMasterFailure.java
Tue Apr  3 02:04:26 2012
@@ -338,7 +338,7 @@ public class TestRegionStateOnMasterFail
           logMsg("Killing master right before it can process the event "
               + eventType + " for region " + openedRegion);
           HBaseEventHandler.unregisterListener(this);
-          localCluster().getActiveMaster().killMaster();
+          miniCluster().killActiveMaster();
           terminateEventThread = true;
         } else {
           logMsg("Skipping event for region " + openedRegion

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java?rev=1308651&r1=1308650&r2=1308651&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
(original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
Tue Apr  3 02:04:26 2012
@@ -25,13 +25,13 @@ import static org.junit.Assert.*;
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.concurrent.Semaphore;
-import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.Stoppable;
 import org.apache.hadoop.hbase.master.SplitLogManager.Task;
 import org.apache.hadoop.hbase.master.SplitLogManager.TaskBatch;
 import org.apache.hadoop.hbase.zookeeper.ZKSplitLog;
@@ -68,7 +68,24 @@ public class TestSplitLogManager {
   private final static HBaseTestingUtility TEST_UTIL =
     new HBaseTestingUtility();
 
-  static AtomicBoolean stopper = new AtomicBoolean();
+  private volatile boolean stopped = false;
+
+  private Stoppable stopper = new Stoppable() {
+    @Override
+    public boolean isStopped() {
+      return stopped;
+    }
+
+    @Override
+    public void stop(String why) {
+      stopped = true;
+    }
+
+    @Override
+    public String getStopReason() {
+      return "test";
+    }
+  };
 
   @BeforeClass
   public static void setUpBeforeClass() throws Exception {
@@ -82,7 +99,6 @@ public class TestSplitLogManager {
 
   @Before
   public void setup() throws Exception {
-    stopper.set(false);
     conf = TEST_UTIL.getConfiguration();
     zkw = ZooKeeperWrapper.createInstance(conf, "split-log-manager-tests");
     zkw.deleteChildrenRecursively(zkw.parentZNode);
@@ -98,7 +114,7 @@ public class TestSplitLogManager {
 
   @After
   public void teardown() throws IOException, KeeperException {
-    stopper.set(true);
+    stopped = true;
     zkw.close();
     slm.stop();
   }



Mime
View raw message