hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From syuanji...@apache.org
Subject [37/50] [abbrv] hbase git commit: HBASE-16634 Speedup TestExportSnapshot
Date Mon, 19 Sep 2016 23:17:50 GMT
HBASE-16634 Speedup TestExportSnapshot


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e19632a1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e19632a1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e19632a1

Branch: refs/heads/hbase-12439
Commit: e19632a19abc52a983d444811852bf0f11cd0d0b
Parents: 216e847
Author: Matteo Bertozzi <matteo.bertozzi@cloudera.com>
Authored: Thu Sep 15 18:29:09 2016 -0700
Committer: Matteo Bertozzi <matteo.bertozzi@cloudera.com>
Committed: Thu Sep 15 18:29:09 2016 -0700

----------------------------------------------------------------------
 .../hbase/snapshot/MobSnapshotTestingUtils.java | 15 ++++++++-
 .../hbase/snapshot/SnapshotTestingUtils.java    | 34 ++++++++++++++++----
 .../hbase/snapshot/TestExportSnapshot.java      |  8 ++---
 .../snapshot/TestFlushSnapshotFromClient.java   |  8 ++---
 .../hbase/snapshot/TestMobExportSnapshot.java   |  8 ++---
 .../snapshot/TestMobSecureExportSnapshot.java   |  4 +--
 .../snapshot/TestSecureExportSnapshot.java      |  1 -
 7 files changed, 54 insertions(+), 24 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/e19632a1/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/MobSnapshotTestingUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/MobSnapshotTestingUtils.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/MobSnapshotTestingUtils.java
index 9026115..107f487 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/MobSnapshotTestingUtils.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/MobSnapshotTestingUtils.java
@@ -38,6 +38,20 @@ public class MobSnapshotTestingUtils {
   public static void createMobTable(final HBaseTestingUtility util,
       final TableName tableName, int regionReplication,
       final byte[]... families) throws IOException, InterruptedException {
+    createMobTable(util, tableName, SnapshotTestingUtils.getSplitKeys(),
+      regionReplication, families);
+  }
+
+  public static void createPreSplitMobTable(final HBaseTestingUtility util,
+      final TableName tableName, int nRegions, final byte[]... families)
+      throws IOException, InterruptedException {
+    createMobTable(util, tableName, SnapshotTestingUtils.getSplitKeys(nRegions),
+      1, families);
+  }
+
+  private static void createMobTable(final HBaseTestingUtility util,
+      final TableName tableName, final byte[][] splitKeys, int regionReplication,
+      final byte[]... families) throws IOException, InterruptedException {
     HTableDescriptor htd = new HTableDescriptor(tableName);
     htd.setRegionReplication(regionReplication);
     for (byte[] family : families) {
@@ -46,7 +60,6 @@ public class MobSnapshotTestingUtils {
       hcd.setMobThreshold(0L);
       htd.addFamily(hcd);
     }
-    byte[][] splitKeys = SnapshotTestingUtils.getSplitKeys();
     util.getHBaseAdmin().createTable(htd, splitKeys);
     SnapshotTestingUtils.waitForTableToBeOnline(util, tableName);
     assertEquals((splitKeys.length + 1) * regionReplication, util

http://git-wip-us.apache.org/repos/asf/hbase/blob/e19632a1/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
index 770bb00..dfd00b3 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
@@ -82,9 +82,10 @@ import com.google.protobuf.ServiceException;
  */
 @InterfaceAudience.Private
 public final class SnapshotTestingUtils {
-
   private static final Log LOG = LogFactory.getLog(SnapshotTestingUtils.class);
-  private static byte[] KEYS = Bytes.toBytes("0123456789");
+
+  // default number of regions (and keys) given by getSplitKeys() and createTable()
+  private static byte[] KEYS = Bytes.toBytes("0123456");
 
   private SnapshotTestingUtils() {
     // private constructor for utility class
@@ -750,23 +751,32 @@ public final class SnapshotTestingUtils {
   }
 
   public static void createTable(final HBaseTestingUtility util, final TableName tableName,
-      int regionReplication, final byte[]... families) throws IOException, InterruptedException
{
+      int regionReplication, int nRegions, final byte[]... families)
+      throws IOException, InterruptedException {
     HTableDescriptor htd = new HTableDescriptor(tableName);
     htd.setRegionReplication(regionReplication);
     for (byte[] family : families) {
       HColumnDescriptor hcd = new HColumnDescriptor(family);
       htd.addFamily(hcd);
     }
-    byte[][] splitKeys = getSplitKeys();
+    byte[][] splitKeys = getSplitKeys(nRegions);
     util.createTable(htd, splitKeys);
     assertEquals((splitKeys.length + 1) * regionReplication,
         util.getHBaseAdmin().getTableRegions(tableName).size());
   }
 
   public static byte[][] getSplitKeys() {
-    byte[][] splitKeys = new byte[KEYS.length-2][];
+    return getSplitKeys(KEYS.length);
+  }
+
+  public static byte[][] getSplitKeys(int nRegions) {
+    nRegions = nRegions < KEYS.length ? nRegions : (KEYS.length - 1);
+    final byte[][] splitKeys = new byte[nRegions-1][];
+    final int step = KEYS.length / nRegions;
+    int keyIndex = 1;
     for (int i = 0; i < splitKeys.length; ++i) {
-      splitKeys[i] = new byte[] { KEYS[i+1] };
+      splitKeys[i] = new byte[] { KEYS[keyIndex] };
+      keyIndex += step;
     }
     return splitKeys;
   }
@@ -776,6 +786,16 @@ public final class SnapshotTestingUtils {
     createTable(util, tableName, 1, families);
   }
 
+  public static void createTable(final HBaseTestingUtility util, final TableName tableName,
+      final int regionReplication, final byte[]... families) throws IOException, InterruptedException
{
+    createTable(util, tableName, regionReplication, KEYS.length, families);
+  }
+
+  public static void createPreSplitTable(final HBaseTestingUtility util, final TableName
tableName,
+      final int nRegions, final byte[]... families) throws IOException, InterruptedException
{
+    createTable(util, tableName, 1, nRegions, families);
+  }
+
   public static void loadData(final HBaseTestingUtility util, final TableName tableName,
int rows,
       byte[]... families) throws IOException, InterruptedException {
     BufferedMutator mutator = util.getConnection().getBufferedMutator(tableName);
@@ -861,6 +881,6 @@ public final class SnapshotTestingUtils {
         }
       }
     }
-    assert(set.size() == getSplitKeys().length + 1);
+    assertEquals(getSplitKeys().length + 1, set.size());
   }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/e19632a1/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
index d0b32f8..ee0d0e2 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
@@ -45,7 +45,7 @@ import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescriptio
 import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo;
 import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
 import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils.SnapshotMock;
-import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
 import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
@@ -62,7 +62,7 @@ import org.junit.rules.TestRule;
 /**
  * Test Export Snapshot Tool
  */
-@Category({VerySlowRegionServerTests.class, MediumTests.class})
+@Category({VerySlowRegionServerTests.class, LargeTests.class})
 public class TestExportSnapshot {
   @Rule public final TestRule timeout = CategoryBasedTimeout.builder().
       withTimeout(this.getClass()).withLookingForStuckThread(true).build();
@@ -91,12 +91,10 @@ public class TestExportSnapshot {
   public static void setUpBeforeClass() throws Exception {
     setUpBaseConf(TEST_UTIL.getConfiguration());
     TEST_UTIL.startMiniCluster(3);
-    TEST_UTIL.startMiniMapReduceCluster();
   }
 
   @AfterClass
   public static void tearDownAfterClass() throws Exception {
-    TEST_UTIL.shutdownMiniMapReduceCluster();
     TEST_UTIL.shutdownMiniCluster();
   }
 
@@ -127,7 +125,7 @@ public class TestExportSnapshot {
   }
 
   protected void createTable() throws Exception {
-    SnapshotTestingUtils.createTable(TEST_UTIL, tableName, FAMILY);
+    SnapshotTestingUtils.createPreSplitTable(TEST_UTIL, tableName, 2, FAMILY);
   }
 
   @After

http://git-wip-us.apache.org/repos/asf/hbase/blob/e19632a1/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
index 8efc2d7..a852547 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
@@ -325,8 +325,8 @@ public class TestFlushSnapshotFromClient {
     int numRegionsAfterMerge = numRegions - 2;
     admin.mergeRegionsAsync(regions.get(1).getEncodedNameAsBytes(),
         regions.get(2).getEncodedNameAsBytes(), true);
-    admin.mergeRegionsAsync(regions.get(5).getEncodedNameAsBytes(),
-        regions.get(6).getEncodedNameAsBytes(), true);
+    admin.mergeRegionsAsync(regions.get(4).getEncodedNameAsBytes(),
+        regions.get(5).getEncodedNameAsBytes(), true);
 
     // Verify that there's one region less
     waitRegionsAfterMerge(numRegionsAfterMerge);
@@ -366,8 +366,8 @@ public class TestFlushSnapshotFromClient {
     int numRegionsAfterMerge = numRegions - 2;
     admin.mergeRegionsAsync(regions.get(1).getEncodedNameAsBytes(),
         regions.get(2).getEncodedNameAsBytes(), true);
-    admin.mergeRegionsAsync(regions.get(5).getEncodedNameAsBytes(),
-        regions.get(6).getEncodedNameAsBytes(), true);
+    admin.mergeRegionsAsync(regions.get(4).getEncodedNameAsBytes(),
+        regions.get(5).getEncodedNameAsBytes(), true);
 
     waitRegionsAfterMerge(numRegionsAfterMerge);
     assertEquals(numRegionsAfterMerge, admin.getTableRegions(TABLE_NAME).size());

http://git-wip-us.apache.org/repos/asf/hbase/blob/e19632a1/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java
index b6cb339..d755961 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java
@@ -25,14 +25,15 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.mob.MobConstants;
 import org.apache.hadoop.hbase.mob.MobUtils;
-import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests;
 import org.junit.BeforeClass;
 import org.junit.experimental.categories.Category;
 
 /**
  * Test Export Snapshot Tool
  */
-@Category(MediumTests.class)
+@Category({VerySlowRegionServerTests.class, LargeTests.class})
 public class TestMobExportSnapshot extends TestExportSnapshot {
   private final Log LOG = LogFactory.getLog(getClass());
 
@@ -45,12 +46,11 @@ public class TestMobExportSnapshot extends TestExportSnapshot {
   public static void setUpBeforeClass() throws Exception {
     setUpBaseConf(TEST_UTIL.getConfiguration());
     TEST_UTIL.startMiniCluster(3);
-    TEST_UTIL.startMiniMapReduceCluster();
   }
 
   @Override
   protected void createTable() throws Exception {
-    MobSnapshotTestingUtils.createMobTable(TEST_UTIL, tableName, 1, FAMILY);
+    MobSnapshotTestingUtils.createPreSplitMobTable(TEST_UTIL, tableName, 2, FAMILY);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hbase/blob/e19632a1/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobSecureExportSnapshot.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobSecureExportSnapshot.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobSecureExportSnapshot.java
index 4a89db8..8154995 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobSecureExportSnapshot.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobSecureExportSnapshot.java
@@ -20,6 +20,7 @@
 package org.apache.hadoop.hbase.snapshot;
 
 import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests;
 import org.apache.hadoop.hbase.mapreduce.HadoopSecurityEnabledUserProviderForTesting;
 import org.apache.hadoop.hbase.security.UserProvider;
 import org.apache.hadoop.hbase.security.access.AccessControlLists;
@@ -31,7 +32,7 @@ import org.junit.experimental.categories.Category;
 /**
  * Reruns TestMobExportSnapshot using MobExportSnapshot in secure mode.
  */
-@Category(LargeTests.class)
+@Category({VerySlowRegionServerTests.class, LargeTests.class})
 public class TestMobSecureExportSnapshot extends TestMobExportSnapshot {
   @BeforeClass
   public static void setUpBeforeClass() throws Exception {
@@ -45,7 +46,6 @@ public class TestMobSecureExportSnapshot extends TestMobExportSnapshot {
     SecureTestUtil.enableSecurity(TEST_UTIL.getConfiguration());
 
     TEST_UTIL.startMiniCluster(3);
-    TEST_UTIL.startMiniMapReduceCluster();
 
     // Wait for the ACL table to become available
     TEST_UTIL.waitTableEnabled(AccessControlLists.ACL_TABLE_NAME);

http://git-wip-us.apache.org/repos/asf/hbase/blob/e19632a1/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshot.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshot.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshot.java
index cf55c4d..f335e4e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshot.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshot.java
@@ -51,7 +51,6 @@ public class TestSecureExportSnapshot extends TestExportSnapshot {
     SecureTestUtil.enableSecurity(TEST_UTIL.getConfiguration());
 
     TEST_UTIL.startMiniCluster(3);
-    TEST_UTIL.startMiniMapReduceCluster();
 
     // Wait for the ACL table to become available
     TEST_UTIL.waitTableEnabled(AccessControlLists.ACL_TABLE_NAME);


Mime
View raw message