hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jeffr...@apache.org
Subject svn commit: r1508316 - /hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
Date Tue, 30 Jul 2013 06:27:18 GMT
Author: jeffreyz
Date: Tue Jul 30 06:27:18 2013
New Revision: 1508316

URL: http://svn.apache.org/r1508316
Log:
HBASE-8960: TestDistributedLogSplitting.testLogReplayForDisablingTable fails sometimes - Addendum

Modified:
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java?rev=1508316&r1=1508315&r2=1508316&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
(original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
Tue Jul 30 06:27:18 2013
@@ -233,37 +233,14 @@ public class TestDistributedLogSplitting
     final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null);
     HTable ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE);
 
-    List<HRegionInfo> regions = null;
-    HRegionServer hrs = null;
-    for (int i = 0; i < NUM_RS; i++) {
-      boolean isCarryingMeta = false;
-      hrs = rsts.get(i).getRegionServer();
-      regions = ProtobufUtil.getOnlineRegions(hrs);
-      for (HRegionInfo region : regions) {
-        if (region.isMetaRegion()) {
-          isCarryingMeta = true;
-          break;
-        }
-      }
-      if (isCarryingMeta) {
-        continue;
-      }
-      break;
-    }
-
-    LOG.info("#regions = " + regions.size());
-    Iterator<HRegionInfo> it = regions.iterator();
-    while (it.hasNext()) {
-      HRegionInfo region = it.next();
-      if (region.isMetaTable()) {
-        it.remove();
-      }
-    }
+    HRegionServer hrs = findRSToKill(false, "table");
+    List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs);
     makeHLog(hrs.getWAL(), regions, "table", "family", NUM_LOG_LINES, 100);
 
     // wait for abort completes
     this.abortRSAndVerifyRecovery(hrs, ht, zkw, NUM_REGIONS_TO_CREATE, NUM_LOG_LINES);
     ht.close();
+    zkw.close();
   }
 
   @Test(timeout = 300000)
@@ -282,36 +259,13 @@ public class TestDistributedLogSplitting
     final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null);
     HTable ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE);
 
-    List<HRegionInfo> regions = null;
-    HRegionServer hrs = null;
-    for (int i = 0; i < NUM_RS; i++) {
-      boolean isCarryingMeta = false;
-      hrs = rsts.get(i).getRegionServer();
-      regions = ProtobufUtil.getOnlineRegions(hrs);
-      for (HRegionInfo region : regions) {
-        if (region.isMetaRegion()) {
-          isCarryingMeta = true;
-          break;
-        }
-      }
-      if (!isCarryingMeta) {
-        continue;
-      }
-      break;
-    }
-
-    LOG.info("#regions = " + regions.size());
-    Iterator<HRegionInfo> it = regions.iterator();
-    while (it.hasNext()) {
-      HRegionInfo region = it.next();
-      if (region.isMetaTable()) {
-        it.remove();
-      }
-    }
+    HRegionServer hrs = findRSToKill(true, "table");
+    List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs);
     makeHLog(hrs.getWAL(), regions, "table", "family", NUM_LOG_LINES, 100);
 
     this.abortRSAndVerifyRecovery(hrs, ht, zkw, NUM_REGIONS_TO_CREATE, NUM_LOG_LINES);
     ht.close();
+    zkw.close();
   }
 
   private void abortRSAndVerifyRecovery(HRegionServer hrs, HTable ht, final ZooKeeperWatcher
zkw,
@@ -374,32 +328,8 @@ public class TestDistributedLogSplitting
     final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null);
     HTable ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE);
 
-    List<HRegionInfo> regions = null;
-    HRegionServer hrs = null;
-    for (int i = 0; i < NUM_RS; i++) {
-      boolean isCarryingMeta = false;
-      hrs = rsts.get(i).getRegionServer();
-      regions = ProtobufUtil.getOnlineRegions(hrs);
-      for (HRegionInfo region : regions) {
-        if (region.isMetaRegion()) {
-          isCarryingMeta = true;
-          break;
-        }
-      }
-      if (isCarryingMeta) {
-        continue;
-      }
-      break;
-    }
-
-    LOG.info("#regions = " + regions.size());
-    Iterator<HRegionInfo> it = regions.iterator();
-    while (it.hasNext()) {
-      HRegionInfo region = it.next();
-      if (region.isMetaTable()) {
-        it.remove();
-      }
-    }
+    HRegionServer hrs = findRSToKill(false, "table");
+    List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs);
     makeHLog(hrs.getWAL(), regions, "table", "family", NUM_LOG_LINES, 100);
 
     // abort master
@@ -436,6 +366,7 @@ public class TestDistributedLogSplitting
     assertEquals(NUM_LOG_LINES, TEST_UTIL.countRows(ht));
 
     ht.close();
+    zkw.close();
   }
   
   @Test(timeout = 300000)
@@ -456,32 +387,8 @@ public class TestDistributedLogSplitting
     final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null);
     HTable ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE);
 
-    List<HRegionInfo> regions = null;
-    HRegionServer hrs = null;
-    for (int i = 0; i < NUM_RS; i++) {
-      boolean isCarryingMeta = false;
-      hrs = rsts.get(i).getRegionServer();
-      regions = ProtobufUtil.getOnlineRegions(hrs);
-      for (HRegionInfo region : regions) {
-        if (region.isMetaRegion()) {
-          isCarryingMeta = true;
-          break;
-        }
-      }
-      if (isCarryingMeta) {
-        continue;
-      }
-      break;
-    }
-
-    LOG.info("#regions = " + regions.size());
-    Iterator<HRegionInfo> it = regions.iterator();
-    while (it.hasNext()) {
-      HRegionInfo region = it.next();
-      if (region.isMetaTable()) {
-        it.remove();
-      }
-    }
+    HRegionServer hrs = findRSToKill(false, "table");
+    List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs);
     makeHLog(hrs.getWAL(), regions, "table", "family", NUM_LOG_LINES, 100);
 
     // abort master
@@ -520,6 +427,7 @@ public class TestDistributedLogSplitting
     assertEquals(NUM_LOG_LINES, TEST_UTIL.countRows(ht));
 
     ht.close();
+    zkw.close();
   }
 
 
@@ -601,6 +509,7 @@ public class TestDistributedLogSplitting
 
     assertEquals(NUM_LOG_LINES, TEST_UTIL.countRows(ht));
     ht.close();
+    zkw.close();
   }
 
   @Test(timeout = 300000)
@@ -648,6 +557,7 @@ public class TestDistributedLogSplitting
       }
     });
     ht.close();
+    zkw.close();
   }
 
   @Test(timeout = 300000)
@@ -680,7 +590,7 @@ public class TestDistributedLogSplitting
       if (isCarryingMeta) {
         continue;
       }
-      break;
+      if (regions.size() > 0) break;
     }
 
     this.prepareData(ht, Bytes.toBytes("family"), Bytes.toBytes("c1"));
@@ -693,6 +603,7 @@ public class TestDistributedLogSplitting
       TEST_UTIL.checksumRows(ht));
 
     ht.close();
+    zkw.close();
   }
 
   @Test(timeout = 300000)
@@ -825,6 +736,7 @@ public class TestDistributedLogSplitting
     }
     disablingHT.close();
     ht.close();
+    zkw.close();
   }
 
   @Test(timeout = 300000)
@@ -845,32 +757,8 @@ public class TestDistributedLogSplitting
     final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null);
     HTable ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE);
 
-    List<HRegionInfo> regions = null;
-    HRegionServer hrs = null;
-    for (int i = 0; i < NUM_RS; i++) {
-      boolean isCarryingMeta = false;
-      hrs = rsts.get(i).getRegionServer();
-      regions = ProtobufUtil.getOnlineRegions(hrs);
-      for (HRegionInfo region : regions) {
-        if (region.isMetaRegion()) {
-          isCarryingMeta = true;
-          break;
-        }
-      }
-      if (isCarryingMeta) {
-        continue;
-      }
-      break;
-    }
-
-    LOG.info("#regions = " + regions.size());
-    Iterator<HRegionInfo> it = regions.iterator();
-    while (it.hasNext()) {
-      HRegionInfo region = it.next();
-      if (region.isMetaTable()) {
-        it.remove();
-      }
-    }
+    HRegionServer hrs = findRSToKill(false, "table");
+    List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs);
     makeHLog(hrs.getWAL(), regions, "table", "family", NUM_LOG_LINES, 100);
     
     // abort RS
@@ -910,6 +798,7 @@ public class TestDistributedLogSplitting
     }
 
     ht.close();
+    zkw.close();
   }
 
   /**
@@ -1028,6 +917,7 @@ public class TestDistributedLogSplitting
     assertEquals(NUM_REGIONS_TO_CREATE * NUM_ROWS_PER_REGION,
         TEST_UTIL.countRows(ht));
     ht.close();
+    zkw.close();
   }
 
 
@@ -1109,24 +999,9 @@ public class TestDistributedLogSplitting
     final ZooKeeperWatcher zkw = new ZooKeeperWatcher(curConf, "table-creation", null);
     List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
 
-    installTable(zkw, "table", "family", 40);
-    List<HRegionInfo> regions = null;
-    HRegionServer hrs = null;
-    for (int i = 0; i < NUM_RS; i++) {
-      boolean isCarryingMeta = false;
-      hrs = rsts.get(i).getRegionServer();
-      regions = ProtobufUtil.getOnlineRegions(hrs);
-      for (HRegionInfo region : regions) {
-        if (region.isMetaRegion()) {
-          isCarryingMeta = true;
-          break;
-        }
-      }
-      if (!isCarryingMeta) {
-        continue;
-      }
-      break;
-    }
+    // only testing meta recovery in ZK operation
+    HRegionServer hrs = findRSToKill(true, null);
+    List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs);
 
     LOG.info("#regions = " + regions.size());
     Set<HRegionInfo> tmpRegions = new HashSet<HRegionInfo>();
@@ -1159,6 +1034,7 @@ public class TestDistributedLogSplitting
     }
     // meta region should be recovered
     assertFalse(isMetaRegionInRecovery);
+    zkw.close();
   }
 
   HTable installTable(ZooKeeperWatcher zkw, String tname, String fname, int nrs) throws Exception
{
@@ -1242,6 +1118,7 @@ public class TestDistributedLogSplitting
       }
       hris.add(region);
     }
+    LOG.info("Creating wal edits across " + hris.size() + " regions.");
     for (int i = 0; i < edit_size; i++) {
       value[i] = (byte) ('a' + (i % 26));
     }
@@ -1386,4 +1263,65 @@ public class TestDistributedLogSplitting
       Thread.sleep(100);
     }
   }
+
+  /**
+   * Find a RS that has regions of a table.
+   * @param hasMetaRegion when true, the returned RS has META region as well
+   * @param tableName
+   * @return
+   * @throws Exception
+   */
+  private HRegionServer findRSToKill(boolean hasMetaRegion, String tableName) throws Exception
{
+    List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
+    int numOfRSs = rsts.size();
+    List<HRegionInfo> regions = null;
+    HRegionServer hrs = null;
+
+    for (int i = 0; i < numOfRSs; i++) {
+      boolean isCarryingMeta = false;
+      boolean foundTableRegion = false;
+      hrs = rsts.get(i).getRegionServer();
+      regions = ProtobufUtil.getOnlineRegions(hrs);
+      for (HRegionInfo region : regions) {
+        if (region.isMetaRegion()) {
+          isCarryingMeta = true;
+        }
+        if (tableName == null || region.getTableNameAsString().equalsIgnoreCase(tableName))
{
+          foundTableRegion = true;
+        }
+        if (foundTableRegion && (isCarryingMeta || !hasMetaRegion)) {
+          break;
+        }
+      }
+      if (isCarryingMeta && hasMetaRegion) {
+        // clients ask for a RS with META
+        if (!foundTableRegion) {
+          final HRegionServer destRS = hrs;
+          // the RS doesn't have regions of the specified table so we need move one to this
RS
+          List<HRegionInfo> tableRegions =
+              TEST_UTIL.getHBaseAdmin().getTableRegions(Bytes.toBytes(tableName));
+          final HRegionInfo hri = tableRegions.get(0);
+          TEST_UTIL.getHBaseAdmin().move(hri.getEncodedNameAsBytes(),
+            Bytes.toBytes(destRS.getServerName().getServerName()));
+          // wait for region move completes
+          final RegionStates regionStates =
+              TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
+          TEST_UTIL.waitFor(45000, 200, new Waiter.Predicate<Exception>() {
+            @Override
+            public boolean evaluate() throws Exception {
+              ServerName sn = regionStates.getRegionServerOfRegion(hri);
+              return (sn != null && sn.equals(destRS.getServerName()));
+            }
+          });
+        }
+        return hrs;
+      } else if (hasMetaRegion || isCarryingMeta) {
+        continue;
+      }
+      if (foundTableRegion) break;
+    }
+
+    return hrs;
+  }
+
 }



Mime
View raw message