Return-Path: X-Original-To: apmail-hbase-commits-archive@www.apache.org Delivered-To: apmail-hbase-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id D16C818C44 for ; Wed, 17 Feb 2016 17:36:10 +0000 (UTC) Received: (qmail 27447 invoked by uid 500); 17 Feb 2016 17:36:06 -0000 Delivered-To: apmail-hbase-commits-archive@hbase.apache.org Received: (qmail 27307 invoked by uid 500); 17 Feb 2016 17:36:06 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 26893 invoked by uid 99); 17 Feb 2016 17:36:06 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 17 Feb 2016 17:36:06 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id DC563E1126; Wed, 17 Feb 2016 17:36:05 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: misty@apache.org To: commits@hbase.apache.org Date: Wed, 17 Feb 2016 17:36:09 -0000 Message-Id: <652cb447e6d848e8bd3652d9393aeb24@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [05/18] hbase-site git commit: Published site at 9f8273e7175954cf6dd4bf523b531bf9971749bb. http://git-wip-us.apache.org/repos/asf/hbase-site/blob/ee77a3b9/xref-test/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.html ---------------------------------------------------------------------- diff --git a/xref-test/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.html b/xref-test/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.html index 29cab65..ce41cb7 100644 --- a/xref-test/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.html +++ b/xref-test/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.html @@ -44,260 +44,260 @@ 34 import org.apache.hadoop.conf.Configuration; 35 import org.apache.hadoop.fs.FileSystem; 36 import org.apache.hadoop.fs.Path; -37 import org.apache.hadoop.hbase.HBaseTestingUtility; -38 import org.apache.hadoop.hbase.HConstants; -39 import org.apache.hadoop.hbase.HRegionInfo; -40 import org.apache.hadoop.hbase.TableName; -41 import org.apache.hadoop.hbase.TableNotFoundException; -42 import org.apache.hadoop.hbase.client.Admin; -43 import org.apache.hadoop.hbase.client.Table; -44 import org.apache.hadoop.hbase.master.HMaster; -45 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; -46 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; -47 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy; -48 import org.apache.hadoop.hbase.testclassification.LargeTests; -49 import org.apache.hadoop.hbase.testclassification.RegionServerTests; -50 import org.apache.hadoop.hbase.util.Bytes; -51 import org.junit.After; -52 import org.junit.AfterClass; -53 import org.junit.Before; -54 import org.junit.BeforeClass; -55 import org.junit.Test; -56 import org.junit.Ignore; -57 import org.junit.experimental.categories.Category; -58 -59 /** -60 * Test creating/using/deleting snapshots from the client -61 * <p> -62 * This is an end-to-end test for the snapshot utility -63 * -64 * TODO This is essentially a clone of TestSnapshotFromClient. This is worth refactoring this -65 * because there will be a few more flavors of snapshots that need to run these tests. -66 */ -67 @Category({RegionServerTests.class, LargeTests.class}) -68 public class TestFlushSnapshotFromClient { -69 private static final Log LOG = LogFactory.getLog(TestFlushSnapshotFromClient.class); -70 -71 protected static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); -72 protected static final int NUM_RS = 2; -73 protected static final byte[] TEST_FAM = Bytes.toBytes("fam"); -74 protected static final TableName TABLE_NAME = TableName.valueOf("test"); -75 protected final int DEFAULT_NUM_ROWS = 100; -76 -77 @BeforeClass -78 public static void setupCluster() throws Exception { -79 setupConf(UTIL.getConfiguration()); -80 UTIL.startMiniCluster(NUM_RS); -81 } -82 -83 protected static void setupConf(Configuration conf) { -84 // disable the ui -85 conf.setInt("hbase.regionsever.info.port", -1); -86 // change the flush size to a small amount, regulating number of store files -87 conf.setInt("hbase.hregion.memstore.flush.size", 25000); -88 // so make sure we get a compaction when doing a load, but keep around some -89 // files in the store -90 conf.setInt("hbase.hstore.compaction.min", 10); -91 conf.setInt("hbase.hstore.compactionThreshold", 10); -92 // block writes if we get to 12 store files -93 conf.setInt("hbase.hstore.blockingStoreFiles", 12); -94 // Enable snapshot -95 conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); -96 conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, -97 ConstantSizeRegionSplitPolicy.class.getName()); -98 } -99 -100 @Before -101 public void setup() throws Exception { -102 createTable(); +37 import org.apache.hadoop.hbase.CategoryBasedTimeout; +38 import org.apache.hadoop.hbase.HBaseTestingUtility; +39 import org.apache.hadoop.hbase.HConstants; +40 import org.apache.hadoop.hbase.HRegionInfo; +41 import org.apache.hadoop.hbase.TableName; +42 import org.apache.hadoop.hbase.TableNotFoundException; +43 import org.apache.hadoop.hbase.client.Admin; +44 import org.apache.hadoop.hbase.client.Table; +45 import org.apache.hadoop.hbase.master.HMaster; +46 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; +47 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; +48 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy; +49 import org.apache.hadoop.hbase.testclassification.LargeTests; +50 import org.apache.hadoop.hbase.testclassification.RegionServerTests; +51 import org.apache.hadoop.hbase.util.Bytes; +52 import org.junit.After; +53 import org.junit.AfterClass; +54 import org.junit.Before; +55 import org.junit.BeforeClass; +56 import org.junit.Rule; +57 import org.junit.Test; +58 import org.junit.experimental.categories.Category; +59 import org.junit.rules.TestRule; +60 +61 /** +62 * Test creating/using/deleting snapshots from the client +63 * <p> +64 * This is an end-to-end test for the snapshot utility +65 * +66 * TODO This is essentially a clone of TestSnapshotFromClient. This is worth refactoring this +67 * because there will be a few more flavors of snapshots that need to run these tests. +68 */ +69 @Category({RegionServerTests.class, LargeTests.class}) +70 public class TestFlushSnapshotFromClient { +71 private static final Log LOG = LogFactory.getLog(TestFlushSnapshotFromClient.class); +72 @Rule public final TestRule timeout = CategoryBasedTimeout.builder().withTimeout(this.getClass()). +73 withLookingForStuckThread(true).build(); +74 +75 protected static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); +76 protected static final int NUM_RS = 2; +77 protected static final byte[] TEST_FAM = Bytes.toBytes("fam"); +78 protected static final TableName TABLE_NAME = TableName.valueOf("test"); +79 protected final int DEFAULT_NUM_ROWS = 100; +80 protected Admin admin = null; +81 +82 @BeforeClass +83 public static void setupCluster() throws Exception { +84 setupConf(UTIL.getConfiguration()); +85 UTIL.startMiniCluster(NUM_RS); +86 } +87 +88 protected static void setupConf(Configuration conf) { +89 // disable the ui +90 conf.setInt("hbase.regionsever.info.port", -1); +91 // change the flush size to a small amount, regulating number of store files +92 conf.setInt("hbase.hregion.memstore.flush.size", 25000); +93 // so make sure we get a compaction when doing a load, but keep around some +94 // files in the store +95 conf.setInt("hbase.hstore.compaction.min", 10); +96 conf.setInt("hbase.hstore.compactionThreshold", 10); +97 // block writes if we get to 12 store files +98 conf.setInt("hbase.hstore.blockingStoreFiles", 12); +99 // Enable snapshot +100 conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); +101 conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, +102 ConstantSizeRegionSplitPolicy.class.getName()); 103 } 104 -105 protected void createTable() throws Exception { -106 SnapshotTestingUtils.createTable(UTIL, TABLE_NAME, TEST_FAM); -107 } -108 -109 @After -110 public void tearDown() throws Exception { -111 UTIL.deleteTable(TABLE_NAME); -112 -113 SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin()); -114 SnapshotTestingUtils.deleteArchiveDirectory(UTIL); -115 } -116 -117 @AfterClass -118 public static void cleanupTest() throws Exception { -119 try { -120 UTIL.shutdownMiniCluster(); -121 } catch (Exception e) { -122 LOG.warn("failure shutting down cluster", e); -123 } -124 } -125 -126 /** -127 * Test simple flush snapshotting a table that is online -128 * @throws Exception -129 */ -130 @Test (timeout=300000) -131 public void testFlushTableSnapshot() throws Exception { -132 Admin admin = UTIL.getHBaseAdmin(); -133 // make sure we don't fail on listing snapshots -134 SnapshotTestingUtils.assertNoSnapshots(admin); -135 -136 // put some stuff in the table -137 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM); -138 -139 LOG.debug("FS state before snapshot:"); -140 UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG); -141 -142 // take a snapshot of the enabled table -143 String snapshotString = "offlineTableSnapshot"; -144 byte[] snapshot = Bytes.toBytes(snapshotString); -145 admin.snapshot(snapshotString, TABLE_NAME, SnapshotDescription.Type.FLUSH); -146 LOG.debug("Snapshot completed."); -147 -148 // make sure we have the snapshot -149 List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, -150 snapshot, TABLE_NAME); -151 -152 // make sure its a valid snapshot -153 LOG.debug("FS state after snapshot:"); -154 UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG); -155 -156 SnapshotTestingUtils.confirmSnapshotValid(UTIL, snapshots.get(0), TABLE_NAME, TEST_FAM); -157 } -158 -159 /** -160 * Test snapshotting a table that is online without flushing -161 * @throws Exception -162 */ -163 @Test(timeout=30000) -164 public void testSkipFlushTableSnapshot() throws Exception { -165 Admin admin = UTIL.getHBaseAdmin(); -166 // make sure we don't fail on listing snapshots -167 SnapshotTestingUtils.assertNoSnapshots(admin); -168 -169 // put some stuff in the table -170 Table table = UTIL.getConnection().getTable(TABLE_NAME); -171 UTIL.loadTable(table, TEST_FAM); -172 UTIL.flush(TABLE_NAME); -173 -174 LOG.debug("FS state before snapshot:"); -175 UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG); -176 -177 // take a snapshot of the enabled table -178 String snapshotString = "skipFlushTableSnapshot"; -179 byte[] snapshot = Bytes.toBytes(snapshotString); -180 admin.snapshot(snapshotString, TABLE_NAME, SnapshotDescription.Type.SKIPFLUSH); -181 LOG.debug("Snapshot completed."); -182 -183 // make sure we have the snapshot -184 List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, -185 snapshot, TABLE_NAME); +105 @Before +106 public void setup() throws Exception { +107 createTable(); +108 this.admin = UTIL.getConnection().getAdmin(); +109 } +110 +111 protected void createTable() throws Exception { +112 SnapshotTestingUtils.createTable(UTIL, TABLE_NAME, TEST_FAM); +113 } +114 +115 @After +116 public void tearDown() throws Exception { +117 UTIL.deleteTable(TABLE_NAME); +118 SnapshotTestingUtils.deleteAllSnapshots(this.admin); +119 this.admin.close(); +120 SnapshotTestingUtils.deleteArchiveDirectory(UTIL); +121 } +122 +123 @AfterClass +124 public static void cleanupTest() throws Exception { +125 try { +126 UTIL.shutdownMiniCluster(); +127 } catch (Exception e) { +128 LOG.warn("failure shutting down cluster", e); +129 } +130 } +131 +132 /** +133 * Test simple flush snapshotting a table that is online +134 * @throws Exception +135 */ +136 @Test +137 public void testFlushTableSnapshot() throws Exception { +138 // make sure we don't fail on listing snapshots +139 SnapshotTestingUtils.assertNoSnapshots(admin); +140 +141 // put some stuff in the table +142 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM); +143 +144 LOG.debug("FS state before snapshot:"); +145 UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG); +146 +147 // take a snapshot of the enabled table +148 String snapshotString = "offlineTableSnapshot"; +149 byte[] snapshot = Bytes.toBytes(snapshotString); +150 admin.snapshot(snapshotString, TABLE_NAME, SnapshotDescription.Type.FLUSH); +151 LOG.debug("Snapshot completed."); +152 +153 // make sure we have the snapshot +154 List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, +155 snapshot, TABLE_NAME); +156 +157 // make sure its a valid snapshot +158 LOG.debug("FS state after snapshot:"); +159 UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG); +160 +161 SnapshotTestingUtils.confirmSnapshotValid(UTIL, snapshots.get(0), TABLE_NAME, TEST_FAM); +162 } +163 +164 /** +165 * Test snapshotting a table that is online without flushing +166 * @throws Exception +167 */ +168 @Test +169 public void testSkipFlushTableSnapshot() throws Exception { +170 // make sure we don't fail on listing snapshots +171 SnapshotTestingUtils.assertNoSnapshots(admin); +172 +173 // put some stuff in the table +174 Table table = UTIL.getConnection().getTable(TABLE_NAME); +175 UTIL.loadTable(table, TEST_FAM); +176 UTIL.flush(TABLE_NAME); +177 +178 LOG.debug("FS state before snapshot:"); +179 UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG); +180 +181 // take a snapshot of the enabled table +182 String snapshotString = "skipFlushTableSnapshot"; +183 byte[] snapshot = Bytes.toBytes(snapshotString); +184 admin.snapshot(snapshotString, TABLE_NAME, SnapshotDescription.Type.SKIPFLUSH); +185 LOG.debug("Snapshot completed."); 186 -187 // make sure its a valid snapshot -188 LOG.debug("FS state after snapshot:"); -189 UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG); +187 // make sure we have the snapshot +188 List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, +189 snapshot, TABLE_NAME); 190 -191 SnapshotTestingUtils.confirmSnapshotValid(UTIL, snapshots.get(0), TABLE_NAME, TEST_FAM); -192 -193 admin.deleteSnapshot(snapshot); -194 snapshots = admin.listSnapshots(); -195 SnapshotTestingUtils.assertNoSnapshots(admin); -196 } -197 -198 -199 /** -200 * Test simple flush snapshotting a table that is online -201 * @throws Exception -202 */ -203 @Test (timeout=300000) -204 public void testFlushTableSnapshotWithProcedure() throws Exception { -205 Admin admin = UTIL.getHBaseAdmin(); -206 // make sure we don't fail on listing snapshots -207 SnapshotTestingUtils.assertNoSnapshots(admin); -208 -209 // put some stuff in the table -210 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM); +191 // make sure its a valid snapshot +192 LOG.debug("FS state after snapshot:"); +193 UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG); +194 +195 SnapshotTestingUtils.confirmSnapshotValid(UTIL, snapshots.get(0), TABLE_NAME, TEST_FAM); +196 +197 admin.deleteSnapshot(snapshot); +198 snapshots = admin.listSnapshots(); +199 SnapshotTestingUtils.assertNoSnapshots(admin); +200 } +201 +202 +203 /** +204 * Test simple flush snapshotting a table that is online +205 * @throws Exception +206 */ +207 @Test +208 public void testFlushTableSnapshotWithProcedure() throws Exception { +209 // make sure we don't fail on listing snapshots +210 SnapshotTestingUtils.assertNoSnapshots(admin); 211 -212 LOG.debug("FS state before snapshot:"); -213 UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG); +212 // put some stuff in the table +213 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM); 214 -215 // take a snapshot of the enabled table -216 String snapshotString = "offlineTableSnapshot"; -217 byte[] snapshot = Bytes.toBytes(snapshotString); -218 Map<String, String> props = new HashMap<String, String>(); -219 props.put("table", TABLE_NAME.getNameAsString()); -220 admin.execProcedure(SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION, -221 snapshotString, props); -222 -223 -224 LOG.debug("Snapshot completed."); +215 LOG.debug("FS state before snapshot:"); +216 UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG); +217 +218 // take a snapshot of the enabled table +219 String snapshotString = "offlineTableSnapshot"; +220 byte[] snapshot = Bytes.toBytes(snapshotString); +221 Map<String, String> props = new HashMap<String, String>(); +222 props.put("table", TABLE_NAME.getNameAsString()); +223 admin.execProcedure(SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION, +224 snapshotString, props); 225 -226 // make sure we have the snapshot -227 List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, -228 snapshot, TABLE_NAME); -229 -230 // make sure its a valid snapshot -231 LOG.debug("FS state after snapshot:"); -232 UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG); -233 -234 SnapshotTestingUtils.confirmSnapshotValid(UTIL, snapshots.get(0), TABLE_NAME, TEST_FAM); -235 } +226 +227 LOG.debug("Snapshot completed."); +228 +229 // make sure we have the snapshot +230 List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, +231 snapshot, TABLE_NAME); +232 +233 // make sure its a valid snapshot +234 LOG.debug("FS state after snapshot:"); +235 UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG); 236 -237 @Test (timeout=300000) -238 public void testSnapshotFailsOnNonExistantTable() throws Exception { -239 Admin admin = UTIL.getHBaseAdmin(); -240 // make sure we don't fail on listing snapshots -241 SnapshotTestingUtils.assertNoSnapshots(admin); -242 TableName tableName = TableName.valueOf("_not_a_table"); -243 -244 // make sure the table doesn't exist -245 boolean fail = false; -246 do { -247 try { -248 admin.getTableDescriptor(tableName); -249 fail = true; -250 LOG.error("Table:" + tableName + " already exists, checking a new name"); -251 tableName = TableName.valueOf(tableName+"!"); -252 } catch (TableNotFoundException e) { -253 fail = false; -254 } -255 } while (fail); -256 -257 // snapshot the non-existant table -258 try { -259 admin.snapshot("fail", tableName, SnapshotDescription.Type.FLUSH); -260 fail("Snapshot succeeded even though there is not table."); -261 } catch (SnapshotCreationException e) { -262 LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage()); -263 } -264 } -265 -266 @Test(timeout = 300000) -267 public void testAsyncFlushSnapshot() throws Exception { -268 Admin admin = UTIL.getHBaseAdmin(); -269 SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("asyncSnapshot") -270 .setTable(TABLE_NAME.getNameAsString()) -271 .setType(SnapshotDescription.Type.FLUSH) -272 .build(); -273 -274 // take the snapshot async -275 admin.takeSnapshotAsync(snapshot); -276 -277 // constantly loop, looking for the snapshot to complete -278 HMaster master = UTIL.getMiniHBaseCluster().getMaster(); -279 SnapshotTestingUtils.waitForSnapshotToComplete(master, snapshot, 200); -280 LOG.info(" === Async Snapshot Completed ==="); -281 UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG); -282 -283 // make sure we get the snapshot -284 SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot); -285 } -286 -287 @Test (timeout=300000) -288 public void testSnapshotStateAfterMerge() throws Exception { -289 int numRows = DEFAULT_NUM_ROWS; -290 Admin admin = UTIL.getHBaseAdmin(); +237 SnapshotTestingUtils.confirmSnapshotValid(UTIL, snapshots.get(0), TABLE_NAME, TEST_FAM); +238 } +239 +240 @Test +241 public void testSnapshotFailsOnNonExistantTable() throws Exception { +242 // make sure we don't fail on listing snapshots +243 SnapshotTestingUtils.assertNoSnapshots(admin); +244 TableName tableName = TableName.valueOf("_not_a_table"); +245 +246 // make sure the table doesn't exist +247 boolean fail = false; +248 do { +249 try { +250 admin.getTableDescriptor(tableName); +251 fail = true; +252 LOG.error("Table:" + tableName + " already exists, checking a new name"); +253 tableName = TableName.valueOf(tableName+"!"); +254 } catch (TableNotFoundException e) { +255 fail = false; +256 } +257 } while (fail); +258 +259 // snapshot the non-existant table +260 try { +261 admin.snapshot("fail", tableName, SnapshotDescription.Type.FLUSH); +262 fail("Snapshot succeeded even though there is not table."); +263 } catch (SnapshotCreationException e) { +264 LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage()); +265 } +266 } +267 +268 @Test +269 public void testAsyncFlushSnapshot() throws Exception { +270 SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("asyncSnapshot") +271 .setTable(TABLE_NAME.getNameAsString()) +272 .setType(SnapshotDescription.Type.FLUSH) +273 .build(); +274 +275 // take the snapshot async +276 admin.takeSnapshotAsync(snapshot); +277 +278 // constantly loop, looking for the snapshot to complete +279 HMaster master = UTIL.getMiniHBaseCluster().getMaster(); +280 SnapshotTestingUtils.waitForSnapshotToComplete(master, snapshot, 200); +281 LOG.info(" === Async Snapshot Completed ==="); +282 UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG); +283 +284 // make sure we get the snapshot +285 SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot); +286 } +287 +288 @Test +289 public void testSnapshotStateAfterMerge() throws Exception { +290 int numRows = DEFAULT_NUM_ROWS; 291 // make sure we don't fail on listing snapshots 292 SnapshotTestingUtils.assertNoSnapshots(admin); 293 // load the table so we have some data @@ -345,199 +345,194 @@ 335 UTIL.deleteTable(cloneBeforeMergeName); 336 } 337 -338 @Test (timeout=300000) +338 @Test 339 public void testTakeSnapshotAfterMerge() throws Exception { 340 int numRows = DEFAULT_NUM_ROWS; -341 Admin admin = UTIL.getHBaseAdmin(); -342 // make sure we don't fail on listing snapshots -343 SnapshotTestingUtils.assertNoSnapshots(admin); -344 // load the table so we have some data -345 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, numRows, TEST_FAM); -346 -347 // Merge two regions -348 List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME); -349 Collections.sort(regions, new Comparator<HRegionInfo>() { -350 public int compare(HRegionInfo r1, HRegionInfo r2) { -351 return Bytes.compareTo(r1.getStartKey(), r2.getStartKey()); -352 } -353 }); -354 -355 int numRegions = admin.getTableRegions(TABLE_NAME).size(); -356 int numRegionsAfterMerge = numRegions - 2; -357 admin.mergeRegions(regions.get(1).getEncodedNameAsBytes(), -358 regions.get(2).getEncodedNameAsBytes(), true); -359 admin.mergeRegions(regions.get(5).getEncodedNameAsBytes(), -360 regions.get(6).getEncodedNameAsBytes(), true); -361 -362 waitRegionsAfterMerge(numRegionsAfterMerge); -363 assertEquals(numRegionsAfterMerge, admin.getTableRegions(TABLE_NAME).size()); -364 -365 // Take a snapshot -366 String snapshotName = "snapshotAfterMerge"; -367 SnapshotTestingUtils.snapshot(admin, snapshotName, TABLE_NAME.getNameAsString(), -368 SnapshotDescription.Type.FLUSH, 3); -369 -370 // Clone the table -371 TableName cloneName = TableName.valueOf("cloneMerge"); -372 admin.cloneSnapshot(snapshotName, cloneName); -373 SnapshotTestingUtils.waitForTableToBeOnline(UTIL, cloneName); -374 -375 verifyRowCount(UTIL, TABLE_NAME, numRows); -376 verifyRowCount(UTIL, cloneName, numRows); -377 -378 // test that we can delete the snapshot -379 UTIL.deleteTable(cloneName); -380 } -381 -382 /** -383 * Basic end-to-end test of simple-flush-based snapshots -384 */ -385 @Test (timeout=300000) -386 public void testFlushCreateListDestroy() throws Exception { -387 LOG.debug("------- Starting Snapshot test -------------"); -388 Admin admin = UTIL.getHBaseAdmin(); -389 // make sure we don't fail on listing snapshots -390 SnapshotTestingUtils.assertNoSnapshots(admin); -391 // load the table so we have some data -392 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM); -393 -394 String snapshotName = "flushSnapshotCreateListDestroy"; -395 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem(); -396 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir(); -397 SnapshotTestingUtils.createSnapshotAndValidate(admin, TABLE_NAME, Bytes.toString(TEST_FAM), -398 snapshotName, rootDir, fs, true); -399 } -400 -401 /** -402 * Demonstrate that we reject snapshot requests if there is a snapshot already running on the -403 * same table currently running and that concurrent snapshots on different tables can both -404 * succeed concurretly. -405 */ -406 @Test(timeout=300000) -407 public void testConcurrentSnapshottingAttempts() throws IOException, InterruptedException { -408 final TableName TABLE2_NAME = TableName.valueOf(TABLE_NAME + "2"); -409 -410 int ssNum = 20; -411 Admin admin = UTIL.getHBaseAdmin(); -412 // make sure we don't fail on listing snapshots -413 SnapshotTestingUtils.assertNoSnapshots(admin); -414 // create second testing table -415 SnapshotTestingUtils.createTable(UTIL, TABLE2_NAME, TEST_FAM); -416 // load the table so we have some data -417 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM); -418 SnapshotTestingUtils.loadData(UTIL, TABLE2_NAME, DEFAULT_NUM_ROWS, TEST_FAM); -419 -420 final CountDownLatch toBeSubmitted = new CountDownLatch(ssNum); -421 // We'll have one of these per thread -422 class SSRunnable implements Runnable { -423 SnapshotDescription ss; -424 SSRunnable(SnapshotDescription ss) { -425 this.ss = ss; -426 } -427 -428 @Override -429 public void run() { -430 try { -431 Admin admin = UTIL.getHBaseAdmin(); -432 LOG.info("Submitting snapshot request: " + ClientSnapshotDescriptionUtils.toString(ss)); -433 admin.takeSnapshotAsync(ss); -434 } catch (Exception e) { -435 LOG.info("Exception during snapshot request: " + ClientSnapshotDescriptionUtils.toString( -436 ss) -437 + ". This is ok, we expect some", e); -438 } -439 LOG.info("Submitted snapshot request: " + ClientSnapshotDescriptionUtils.toString(ss)); -440 toBeSubmitted.countDown(); -441 } -442 }; -443 -444 // build descriptions -445 SnapshotDescription[] descs = new SnapshotDescription[ssNum]; -446 for (int i = 0; i < ssNum; i++) { -447 SnapshotDescription.Builder builder = SnapshotDescription.newBuilder(); -448 builder.setTable(((i % 2) == 0 ? TABLE_NAME : TABLE2_NAME).getNameAsString()); -449 builder.setName("ss"+i); -450 builder.setType(SnapshotDescription.Type.FLUSH); -451 descs[i] = builder.build(); -452 } -453 -454 // kick each off its own thread -455 for (int i=0 ; i < ssNum; i++) { -456 new Thread(new SSRunnable(descs[i])).start(); -457 } -458 -459 // wait until all have been submitted -460 toBeSubmitted.await(); -461 -462 // loop until all are done. -463 while (true) { -464 int doneCount = 0; -465 for (SnapshotDescription ss : descs) { -466 try { -467 if (admin.isSnapshotFinished(ss)) { -468 doneCount++; -469 } -470 } catch (Exception e) { -471 LOG.warn("Got an exception when checking for snapshot " + ss.getName(), e); -472 doneCount++; -473 } -474 } -475 if (doneCount == descs.length) { -476 break; -477 } -478 Thread.sleep(100); -479 } -480 -481 // dump for debugging -482 UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG); -483 -484 List<SnapshotDescription> taken = admin.listSnapshots(); -485 int takenSize = taken.size(); -486 LOG.info("Taken " + takenSize + " snapshots: " + taken); -487 assertTrue("We expect at least 1 request to be rejected because of we concurrently" + -488 " issued many requests", takenSize < ssNum && takenSize > 0); -489 -490 // Verify that there's at least one snapshot per table -491 int t1SnapshotsCount = 0; -492 int t2SnapshotsCount = 0; -493 for (SnapshotDescription ss : taken) { -494 if (TableName.valueOf(ss.getTable()).equals(TABLE_NAME)) { -495 t1SnapshotsCount++; -496 } else if (TableName.valueOf(ss.getTable()).equals(TABLE2_NAME)) { -497 t2SnapshotsCount++; -498 } -499 } -500 assertTrue("We expect at least 1 snapshot of table1 ", t1SnapshotsCount > 0); -501 assertTrue("We expect at least 1 snapshot of table2 ", t2SnapshotsCount > 0); -502 -503 UTIL.deleteTable(TABLE2_NAME); -504 } -505 -506 private void waitRegionsAfterMerge(final long numRegionsAfterMerge) -507 throws IOException, InterruptedException { -508 Admin admin = UTIL.getHBaseAdmin(); -509 // Verify that there's one region less -510 long startTime = System.currentTimeMillis(); -511 while (admin.getTableRegions(TABLE_NAME).size() != numRegionsAfterMerge) { -512 // This may be flaky... if after 15sec the merge is not complete give up -513 // it will fail in the assertEquals(numRegionsAfterMerge). -514 if ((System.currentTimeMillis() - startTime) > 15000) -515 break; -516 Thread.sleep(100); -517 } -518 SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TABLE_NAME); -519 } -520 +341 // make sure we don't fail on listing snapshots +342 SnapshotTestingUtils.assertNoSnapshots(admin); +343 // load the table so we have some data +344 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, numRows, TEST_FAM); +345 +346 // Merge two regions +347 List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME); +348 Collections.sort(regions, new Comparator<HRegionInfo>() { +349 public int compare(HRegionInfo r1, HRegionInfo r2) { +350 return Bytes.compareTo(r1.getStartKey(), r2.getStartKey()); +351 } +352 }); +353 +354 int numRegions = admin.getTableRegions(TABLE_NAME).size(); +355 int numRegionsAfterMerge = numRegions - 2; +356 admin.mergeRegions(regions.get(1).getEncodedNameAsBytes(), +357 regions.get(2).getEncodedNameAsBytes(), true); +358 admin.mergeRegions(regions.get(5).getEncodedNameAsBytes(), +359 regions.get(6).getEncodedNameAsBytes(), true); +360 +361 waitRegionsAfterMerge(numRegionsAfterMerge); +362 assertEquals(numRegionsAfterMerge, admin.getTableRegions(TABLE_NAME).size()); +363 +364 // Take a snapshot +365 String snapshotName = "snapshotAfterMerge"; +366 SnapshotTestingUtils.snapshot(admin, snapshotName, TABLE_NAME.getNameAsString(), +367 SnapshotDescription.Type.FLUSH, 3); +368 +369 // Clone the table +370 TableName cloneName = TableName.valueOf("cloneMerge"); +371 admin.cloneSnapshot(snapshotName, cloneName); +372 SnapshotTestingUtils.waitForTableToBeOnline(UTIL, cloneName); +373 +374 verifyRowCount(UTIL, TABLE_NAME, numRows); +375 verifyRowCount(UTIL, cloneName, numRows); +376 +377 // test that we can delete the snapshot +378 UTIL.deleteTable(cloneName); +379 } +380 +381 /** +382 * Basic end-to-end test of simple-flush-based snapshots +383 */ +384 @Test +385 public void testFlushCreateListDestroy() throws Exception { +