Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 03219200C74 for ; Sat, 29 Apr 2017 16:58:42 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 01A0A160BBE; Sat, 29 Apr 2017 14:58:42 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id A2442160BA9 for ; Sat, 29 Apr 2017 16:58:39 +0200 (CEST) Received: (qmail 20769 invoked by uid 500); 29 Apr 2017 14:58:38 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 20750 invoked by uid 99); 29 Apr 2017 14:58:38 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 29 Apr 2017 14:58:38 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 8CC4EE01BC; Sat, 29 Apr 2017 14:58:38 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: git-site-role@apache.org To: commits@hbase.apache.org Date: Sat, 29 Apr 2017 14:58:39 -0000 Message-Id: <75cd4dc48fe347068fa940ea0f09318c@git.apache.org> In-Reply-To: <583f7baa0f14489b97b65a71735dedf3@git.apache.org> References: <583f7baa0f14489b97b65a71735dedf3@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [02/41] hbase-site git commit: Published site at 82d554e3783372cc6b05489452c815b57c06f6cd. archived-at: Sat, 29 Apr 2017 14:58:42 -0000 http://git-wip-us.apache.org/repos/asf/hbase-site/blob/6fbeb554/testdevapidocs/src-html/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.html ---------------------------------------------------------------------- diff --git a/testdevapidocs/src-html/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.html b/testdevapidocs/src-html/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.html index 52d93ac..093bcdb 100644 --- a/testdevapidocs/src-html/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.html +++ b/testdevapidocs/src-html/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.html @@ -28,8 +28,8 @@ 020 021import static org.apache.hadoop.util.ToolRunner.run; 022import static org.junit.Assert.assertEquals; -023import static org.junit.Assert.assertTrue; -024import static org.junit.Assert.assertFalse; +023import static org.junit.Assert.assertFalse; +024import static org.junit.Assert.assertTrue; 025 026import java.io.IOException; 027import java.net.URI; @@ -51,9 +51,9 @@ 043import org.apache.hadoop.hbase.TableName; 044import org.apache.hadoop.hbase.client.Admin; 045import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; -046import org.apache.hadoop.hbase.testclassification.LargeTests; -047import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription; -048import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest; +046import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription; +047import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest; +048import org.apache.hadoop.hbase.testclassification.LargeTests; 049import org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests; 050import org.apache.hadoop.hbase.util.Bytes; 051import org.apache.hadoop.hbase.util.FSUtils; @@ -104,292 +104,287 @@ 096 @BeforeClass 097 public static void setUpBeforeClass() throws Exception { 098 setUpBaseConf(TEST_UTIL.getConfiguration()); -099 // Setup separate test-data directory for MR cluster and set corresponding configurations. -100 // Otherwise, different test classes running MR cluster can step on each other. -101 TEST_UTIL.getDataTestDir(); -102 TEST_UTIL.startMiniZKCluster(); -103 TEST_UTIL.startMiniMapReduceCluster(); -104 TEST_UTIL.startMiniHBaseCluster(1, 3); -105 } -106 -107 @AfterClass -108 public static void tearDownAfterClass() throws Exception { -109 TEST_UTIL.shutdownMiniHBaseCluster(); -110 TEST_UTIL.shutdownMiniMapReduceCluster(); -111 TEST_UTIL.shutdownMiniZKCluster(); -112 } -113 -114 /** -115 * Create a table and take a snapshot of the table used by the export test. -116 */ -117 @Before -118 public void setUp() throws Exception { -119 this.admin = TEST_UTIL.getAdmin(); -120 -121 tableName = TableName.valueOf("testtb-" + testName.getMethodName()); -122 snapshotName = Bytes.toBytes("snaptb0-" + testName.getMethodName()); -123 emptySnapshotName = Bytes.toBytes("emptySnaptb0-" + testName.getMethodName()); -124 -125 // create Table -126 createTable(); -127 -128 // Take an empty snapshot -129 admin.snapshot(emptySnapshotName, tableName); -130 -131 // Add some rows -132 SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 50, FAMILY); -133 tableNumFiles = admin.getTableRegions(tableName).size(); -134 -135 // take a snapshot -136 admin.snapshot(snapshotName, tableName); -137 } -138 -139 protected void createTable() throws Exception { -140 SnapshotTestingUtils.createPreSplitTable(TEST_UTIL, tableName, 2, FAMILY); -141 } -142 -143 protected interface RegionPredicate { -144 boolean evaluate(final HRegionInfo regionInfo); -145 } -146 -147 protected RegionPredicate getBypassRegionPredicate() { -148 return null; -149 } -150 -151 @After -152 public void tearDown() throws Exception { -153 TEST_UTIL.deleteTable(tableName); -154 SnapshotTestingUtils.deleteAllSnapshots(TEST_UTIL.getAdmin()); -155 SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL); -156 } -157 -158 /** -159 * Verify if exported snapshot and copied files matches the original one. -160 */ +099 TEST_UTIL.startMiniCluster(1, 3); +100 TEST_UTIL.startMiniMapReduceCluster(); +101 } +102 +103 @AfterClass +104 public static void tearDownAfterClass() throws Exception { +105 TEST_UTIL.shutdownMiniMapReduceCluster(); +106 TEST_UTIL.shutdownMiniCluster(); +107 } +108 +109 /** +110 * Create a table and take a snapshot of the table used by the export test. +111 */ +112 @Before +113 public void setUp() throws Exception { +114 this.admin = TEST_UTIL.getAdmin(); +115 +116 tableName = TableName.valueOf("testtb-" + testName.getMethodName()); +117 snapshotName = Bytes.toBytes("snaptb0-" + testName.getMethodName()); +118 emptySnapshotName = Bytes.toBytes("emptySnaptb0-" + testName.getMethodName()); +119 +120 // create Table +121 createTable(); +122 +123 // Take an empty snapshot +124 admin.snapshot(emptySnapshotName, tableName); +125 +126 // Add some rows +127 SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 50, FAMILY); +128 tableNumFiles = admin.getTableRegions(tableName).size(); +129 +130 // take a snapshot +131 admin.snapshot(snapshotName, tableName); +132 } +133 +134 protected void createTable() throws Exception { +135 SnapshotTestingUtils.createPreSplitTable(TEST_UTIL, tableName, 2, FAMILY); +136 } +137 +138 protected interface RegionPredicate { +139 boolean evaluate(final HRegionInfo regionInfo); +140 } +141 +142 protected RegionPredicate getBypassRegionPredicate() { +143 return null; +144 } +145 +146 @After +147 public void tearDown() throws Exception { +148 TEST_UTIL.deleteTable(tableName); +149 SnapshotTestingUtils.deleteAllSnapshots(TEST_UTIL.getAdmin()); +150 SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL); +151 } +152 +153 /** +154 * Verify if exported snapshot and copied files matches the original one. +155 */ +156 @Test +157 public void testExportFileSystemState() throws Exception { +158 testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles); +159 } +160 161 @Test -162 public void testExportFileSystemState() throws Exception { -163 testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles); -164 } -165 -166 @Test -167 public void testExportFileSystemStateWithSkipTmp() throws Exception { -168 TEST_UTIL.getConfiguration().setBoolean(ExportSnapshot.CONF_SKIP_TMP, true); -169 try { -170 testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles); -171 } finally { -172 TEST_UTIL.getConfiguration().setBoolean(ExportSnapshot.CONF_SKIP_TMP, false); -173 } +162 public void testExportFileSystemStateWithSkipTmp() throws Exception { +163 TEST_UTIL.getConfiguration().setBoolean(ExportSnapshot.CONF_SKIP_TMP, true); +164 try { +165 testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles); +166 } finally { +167 TEST_UTIL.getConfiguration().setBoolean(ExportSnapshot.CONF_SKIP_TMP, false); +168 } +169 } +170 +171 @Test +172 public void testEmptyExportFileSystemState() throws Exception { +173 testExportFileSystemState(tableName, emptySnapshotName, emptySnapshotName, 0); 174 } 175 176 @Test -177 public void testEmptyExportFileSystemState() throws Exception { -178 testExportFileSystemState(tableName, emptySnapshotName, emptySnapshotName, 0); -179 } -180 -181 @Test -182 public void testConsecutiveExports() throws Exception { -183 Path copyDir = getLocalDestinationDir(); -184 testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles, copyDir, false); -185 testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles, copyDir, true); -186 removeExportDir(copyDir); -187 } -188 -189 @Test -190 public void testExportWithTargetName() throws Exception { -191 final byte[] targetName = Bytes.toBytes("testExportWithTargetName"); -192 testExportFileSystemState(tableName, snapshotName, targetName, tableNumFiles); -193 } -194 -195 private void testExportFileSystemState(final TableName tableName, final byte[] snapshotName, -196 final byte[] targetName, int filesExpected) throws Exception { -197 testExportFileSystemState(tableName, snapshotName, targetName, -198 filesExpected, getHdfsDestinationDir(), false); -199 } -200 -201 protected void testExportFileSystemState(final TableName tableName, -202 final byte[] snapshotName, final byte[] targetName, int filesExpected, -203 Path copyDir, boolean overwrite) throws Exception { -204 testExportFileSystemState(TEST_UTIL.getConfiguration(), tableName, snapshotName, targetName, -205 filesExpected, TEST_UTIL.getDefaultRootDirPath(), copyDir, -206 overwrite, getBypassRegionPredicate(), true); -207 } -208 -209 /** -210 * Creates destination directory, runs ExportSnapshot() tool, and runs some verifications. -211 */ -212 protected static void testExportFileSystemState(final Configuration conf, final TableName tableName, -213 final byte[] snapshotName, final byte[] targetName, final int filesExpected, -214 final Path sourceDir, Path copyDir, final boolean overwrite, -215 final RegionPredicate bypassregionPredicate, boolean success) throws Exception { -216 URI hdfsUri = FileSystem.get(conf).getUri(); -217 FileSystem fs = FileSystem.get(copyDir.toUri(), new Configuration()); -218 copyDir = copyDir.makeQualified(fs); -219 -220 List<String> opts = new ArrayList<>(); -221 opts.add("--snapshot"); -222 opts.add(Bytes.toString(snapshotName)); -223 opts.add("--copy-to"); -224 opts.add(copyDir.toString()); -225 if (targetName != snapshotName) { -226 opts.add("--target"); -227 opts.add(Bytes.toString(targetName)); -228 } -229 if (overwrite) opts.add("--overwrite"); -230 -231 // Export Snapshot -232 int res = run(conf, new ExportSnapshot(), opts.toArray(new String[opts.size()])); -233 assertEquals(success ? 0 : 1, res); -234 if (!success) { -235 final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(targetName)); -236 assertFalse(fs.exists(new Path(copyDir, targetDir))); -237 return; -238 } -239 -240 // Verify File-System state -241 FileStatus[] rootFiles = fs.listStatus(copyDir); -242 assertEquals(filesExpected > 0 ? 2 : 1, rootFiles.length); -243 for (FileStatus fileStatus: rootFiles) { -244 String name = fileStatus.getPath().getName(); -245 assertTrue(fileStatus.isDirectory()); -246 assertTrue(name.equals(HConstants.SNAPSHOT_DIR_NAME) || -247 name.equals(HConstants.HFILE_ARCHIVE_DIRECTORY)); -248 } -249 -250 // compare the snapshot metadata and verify the hfiles -251 final FileSystem hdfs = FileSystem.get(hdfsUri, conf); -252 final Path snapshotDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(snapshotName)); -253 final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(targetName)); -254 verifySnapshotDir(hdfs, new Path(sourceDir, snapshotDir), -255 fs, new Path(copyDir, targetDir)); -256 Set<String> snapshotFiles = verifySnapshot(conf, fs, copyDir, tableName, -257 Bytes.toString(targetName), bypassregionPredicate); -258 assertEquals(filesExpected, snapshotFiles.size()); -259 } -260 -261 /** -262 * Check that ExportSnapshot will succeed if something fails but the retry succeed. -263 */ -264 @Test -265 public void testExportRetry() throws Exception { -266 Path copyDir = getLocalDestinationDir(); -267 FileSystem fs = FileSystem.get(copyDir.toUri(), new Configuration()); -268 copyDir = copyDir.makeQualified(fs); -269 Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); -270 conf.setBoolean(ExportSnapshot.Testing.CONF_TEST_FAILURE, true); -271 conf.setInt(ExportSnapshot.Testing.CONF_TEST_FAILURE_COUNT, 2); -272 conf.setInt("mapreduce.map.maxattempts", 3); -273 testExportFileSystemState(conf, tableName, snapshotName, snapshotName, tableNumFiles, -274 TEST_UTIL.getDefaultRootDirPath(), copyDir, true, getBypassRegionPredicate(), true); -275 } -276 -277 /** -278 * Check that ExportSnapshot will fail if we inject failure more times than MR will retry. -279 */ -280 @Test -281 public void testExportFailure() throws Exception { -282 Path copyDir = getLocalDestinationDir(); -283 FileSystem fs = FileSystem.get(copyDir.toUri(), new Configuration()); -284 copyDir = copyDir.makeQualified(fs); -285 Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); -286 conf.setBoolean(ExportSnapshot.Testing.CONF_TEST_FAILURE, true); -287 conf.setInt(ExportSnapshot.Testing.CONF_TEST_FAILURE_COUNT, 4); -288 conf.setInt("mapreduce.map.maxattempts", 3); -289 testExportFileSystemState(conf, tableName, snapshotName, snapshotName, tableNumFiles, -290 TEST_UTIL.getDefaultRootDirPath(), copyDir, true, getBypassRegionPredicate(), false); -291 } -292 -293 /* -294 * verify if the snapshot folder on file-system 1 match the one on file-system 2 -295 */ -296 protected static void verifySnapshotDir(final FileSystem fs1, final Path root1, -297 final FileSystem fs2, final Path root2) throws IOException { -298 assertEquals(listFiles(fs1, root1, root1), listFiles(fs2, root2, root2)); -299 } -300 -301 protected Set<String> verifySnapshot(final FileSystem fs, final Path rootDir, -302 final TableName tableName, final String snapshotName) throws IOException { -303 return verifySnapshot(TEST_UTIL.getConfiguration(), fs, rootDir, tableName, -304 snapshotName, getBypassRegionPredicate()); -305 } -306 -307 /* -308 * Verify if the files exists -309 */ -310 protected static Set<String> verifySnapshot(final Configuration conf, final FileSystem fs, -311 final Path rootDir, final TableName tableName, final String snapshotName, -312 final RegionPredicate bypassregionPredicate) throws IOException { -313 final Path exportedSnapshot = new Path(rootDir, -314 new Path(HConstants.SNAPSHOT_DIR_NAME, snapshotName)); -315 final Set<String> snapshotFiles = new HashSet<>(); -316 final Path exportedArchive = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY); -317 SnapshotReferenceUtil.visitReferencedFiles(conf, fs, exportedSnapshot, -318 new SnapshotReferenceUtil.SnapshotVisitor() { -319 @Override -320 public void storeFile(final HRegionInfo regionInfo, final String family, -321 final SnapshotRegionManifest.StoreFile storeFile) throws IOException { -322 if (bypassregionPredicate != null && bypassregionPredicate.evaluate(regionInfo)) -323 return; -324 -325 String hfile = storeFile.getName(); -326 snapshotFiles.add(hfile); -327 if (storeFile.hasReference()) { -328 // Nothing to do here, we have already the reference embedded -329 } else { -330 verifyNonEmptyFile(new Path(exportedArchive, -331 new Path(FSUtils.getTableDir(new Path("./"), tableName), -332 new Path(regionInfo.getEncodedName(), new Path(family, hfile))))); -333 } +177 public void testConsecutiveExports() throws Exception { +178 Path copyDir = getLocalDestinationDir(); +179 testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles, copyDir, false); +180 testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles, copyDir, true); +181 removeExportDir(copyDir); +182 } +183 +184 @Test +185 public void testExportWithTargetName() throws Exception { +186 final byte[] targetName = Bytes.toBytes("testExportWithTargetName"); +187 testExportFileSystemState(tableName, snapshotName, targetName, tableNumFiles); +188 } +189 +190 private void testExportFileSystemState(final TableName tableName, final byte[] snapshotName, +191 final byte[] targetName, int filesExpected) throws Exception { +192 testExportFileSystemState(tableName, snapshotName, targetName, +193 filesExpected, getHdfsDestinationDir(), false); +194 } +195 +196 protected void testExportFileSystemState(final TableName tableName, +197 final byte[] snapshotName, final byte[] targetName, int filesExpected, +198 Path copyDir, boolean overwrite) throws Exception { +199 testExportFileSystemState(TEST_UTIL.getConfiguration(), tableName, snapshotName, targetName, +200 filesExpected, TEST_UTIL.getDefaultRootDirPath(), copyDir, +201 overwrite, getBypassRegionPredicate(), true); +202 } +203 +204 /** +205 * Creates destination directory, runs ExportSnapshot() tool, and runs some verifications. +206 */ +207 protected static void testExportFileSystemState(final Configuration conf, final TableName tableName, +208 final byte[] snapshotName, final byte[] targetName, final int filesExpected, +209 final Path sourceDir, Path copyDir, final boolean overwrite, +210 final RegionPredicate bypassregionPredicate, boolean success) throws Exception { +211 URI hdfsUri = FileSystem.get(conf).getUri(); +212 FileSystem fs = FileSystem.get(copyDir.toUri(), new Configuration()); +213 copyDir = copyDir.makeQualified(fs); +214 +215 List<String> opts = new ArrayList<>(); +216 opts.add("--snapshot"); +217 opts.add(Bytes.toString(snapshotName)); +218 opts.add("--copy-to"); +219 opts.add(copyDir.toString()); +220 if (targetName != snapshotName) { +221 opts.add("--target"); +222 opts.add(Bytes.toString(targetName)); +223 } +224 if (overwrite) opts.add("--overwrite"); +225 +226 // Export Snapshot +227 int res = run(conf, new ExportSnapshot(), opts.toArray(new String[opts.size()])); +228 assertEquals(success ? 0 : 1, res); +229 if (!success) { +230 final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(targetName)); +231 assertFalse(fs.exists(new Path(copyDir, targetDir))); +232 return; +233 } +234 +235 // Verify File-System state +236 FileStatus[] rootFiles = fs.listStatus(copyDir); +237 assertEquals(filesExpected > 0 ? 2 : 1, rootFiles.length); +238 for (FileStatus fileStatus: rootFiles) { +239 String name = fileStatus.getPath().getName(); +240 assertTrue(fileStatus.isDirectory()); +241 assertTrue(name.equals(HConstants.SNAPSHOT_DIR_NAME) || +242 name.equals(HConstants.HFILE_ARCHIVE_DIRECTORY)); +243 } +244 +245 // compare the snapshot metadata and verify the hfiles +246 final FileSystem hdfs = FileSystem.get(hdfsUri, conf); +247 final Path snapshotDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(snapshotName)); +248 final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(targetName)); +249 verifySnapshotDir(hdfs, new Path(sourceDir, snapshotDir), +250 fs, new Path(copyDir, targetDir)); +251 Set<String> snapshotFiles = verifySnapshot(conf, fs, copyDir, tableName, +252 Bytes.toString(targetName), bypassregionPredicate); +253 assertEquals(filesExpected, snapshotFiles.size()); +254 } +255 +256 /** +257 * Check that ExportSnapshot will succeed if something fails but the retry succeed. +258 */ +259 @Test +260 public void testExportRetry() throws Exception { +261 Path copyDir = getLocalDestinationDir(); +262 FileSystem fs = FileSystem.get(copyDir.toUri(), new Configuration()); +263 copyDir = copyDir.makeQualified(fs); +264 Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); +265 conf.setBoolean(ExportSnapshot.Testing.CONF_TEST_FAILURE, true); +266 conf.setInt(ExportSnapshot.Testing.CONF_TEST_FAILURE_COUNT, 2); +267 conf.setInt("mapreduce.map.maxattempts", 3); +268 testExportFileSystemState(conf, tableName, snapshotName, snapshotName, tableNumFiles, +269 TEST_UTIL.getDefaultRootDirPath(), copyDir, true, getBypassRegionPredicate(), true); +270 } +271 +272 /** +273 * Check that ExportSnapshot will fail if we inject failure more times than MR will retry. +274 */ +275 @Test +276 public void testExportFailure() throws Exception { +277 Path copyDir = getLocalDestinationDir(); +278 FileSystem fs = FileSystem.get(copyDir.toUri(), new Configuration()); +279 copyDir = copyDir.makeQualified(fs); +280 Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); +281 conf.setBoolean(ExportSnapshot.Testing.CONF_TEST_FAILURE, true); +282 conf.setInt(ExportSnapshot.Testing.CONF_TEST_FAILURE_COUNT, 4); +283 conf.setInt("mapreduce.map.maxattempts", 3); +284 testExportFileSystemState(conf, tableName, snapshotName, snapshotName, tableNumFiles, +285 TEST_UTIL.getDefaultRootDirPath(), copyDir, true, getBypassRegionPredicate(), false); +286 } +287 +288 /* +289 * verify if the snapshot folder on file-system 1 match the one on file-system 2 +290 */ +291 protected static void verifySnapshotDir(final FileSystem fs1, final Path root1, +292 final FileSystem fs2, final Path root2) throws IOException { +293 assertEquals(listFiles(fs1, root1, root1), listFiles(fs2, root2, root2)); +294 } +295 +296 protected Set<String> verifySnapshot(final FileSystem fs, final Path rootDir, +297 final TableName tableName, final String snapshotName) throws IOException { +298 return verifySnapshot(TEST_UTIL.getConfiguration(), fs, rootDir, tableName, +299 snapshotName, getBypassRegionPredicate()); +300 } +301 +302 /* +303 * Verify if the files exists +304 */ +305 protected static Set<String> verifySnapshot(final Configuration conf, final FileSystem fs, +306 final Path rootDir, final TableName tableName, final String snapshotName, +307 final RegionPredicate bypassregionPredicate) throws IOException { +308 final Path exportedSnapshot = new Path(rootDir, +309 new Path(HConstants.SNAPSHOT_DIR_NAME, snapshotName)); +310 final Set<String> snapshotFiles = new HashSet<>(); +311 final Path exportedArchive = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY); +312 SnapshotReferenceUtil.visitReferencedFiles(conf, fs, exportedSnapshot, +313 new SnapshotReferenceUtil.SnapshotVisitor() { +314 @Override +315 public void storeFile(final HRegionInfo regionInfo, final String family, +316 final SnapshotRegionManifest.StoreFile storeFile) throws IOException { +317 if (bypassregionPredicate != null && bypassregionPredicate.evaluate(regionInfo)) +318 return; +319 +320 String hfile = storeFile.getName(); +321 snapshotFiles.add(hfile); +322 if (storeFile.hasReference()) { +323 // Nothing to do here, we have already the reference embedded +324 } else { +325 verifyNonEmptyFile(new Path(exportedArchive, +326 new Path(FSUtils.getTableDir(new Path("./"), tableName), +327 new Path(regionInfo.getEncodedName(), new Path(family, hfile))))); +328 } +329 } +330 +331 private void verifyNonEmptyFile(final Path path) throws IOException { +332 assertTrue(path + " should exists", fs.exists(path)); +333 assertTrue(path + " should not be empty", fs.getFileStatus(path).getLen() > 0); 334 } -335 -336 private void verifyNonEmptyFile(final Path path) throws IOException { -337 assertTrue(path + " should exists", fs.exists(path)); -338 assertTrue(path + " should not be empty", fs.getFileStatus(path).getLen() > 0); -339 } -340 }); -341 -342 // Verify Snapshot description -343 SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, exportedSnapshot); -344 assertTrue(desc.getName().equals(snapshotName)); -345 assertTrue(desc.getTable().equals(tableName.getNameAsString())); -346 return snapshotFiles; -347 } -348 -349 private static Set<String> listFiles(final FileSystem fs, final Path root, final Path dir) -350 throws IOException { -351 Set<String> files = new HashSet<>(); -352 int rootPrefix = root.makeQualified(fs).toString().length(); -353 FileStatus[] list = FSUtils.listStatus(fs, dir); -354 if (list != null) { -355 for (FileStatus fstat: list) { -356 LOG.debug(fstat.getPath()); -357 if (fstat.isDirectory()) { -358 files.addAll(listFiles(fs, root, fstat.getPath())); -359 } else { -360 files.add(fstat.getPath().makeQualified(fs).toString().substring(rootPrefix)); -361 } -362 } -363 } -364 return files; -365 } -366 -367 private Path getHdfsDestinationDir() { -368 Path rootDir = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir(); -369 Path path = new Path(new Path(rootDir, "export-test"), "export-" + System.currentTimeMillis()); -370 LOG.info("HDFS export destination path: " + path); -371 return path; -372 } -373 -374 private Path getLocalDestinationDir() { -375 Path path = TEST_UTIL.getDataTestDir("local-export-" + System.currentTimeMillis()); -376 LOG.info("Local export destination path: " + path); -377 return path; +335 }); +336 +337 // Verify Snapshot description +338 SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, exportedSnapshot); +339 assertTrue(desc.getName().equals(snapshotName)); +340 assertTrue(desc.getTable().equals(tableName.getNameAsString())); +341 return snapshotFiles; +342 } +343 +344 private static Set<String> listFiles(final FileSystem fs, final Path root, final Path dir) +345 throws IOException { +346 Set<String> files = new HashSet<>(); +347 int rootPrefix = root.makeQualified(fs).toString().length(); +348 FileStatus[] list = FSUtils.listStatus(fs, dir); +349 if (list != null) { +350 for (FileStatus fstat: list) { +351 LOG.debug(fstat.getPath()); +352 if (fstat.isDirectory()) { +353 files.addAll(listFiles(fs, root, fstat.getPath())); +354 } else { +355 files.add(fstat.getPath().makeQualified(fs).toString().substring(rootPrefix)); +356 } +357 } +358 } +359 return files; +360 } +361 +362 private Path getHdfsDestinationDir() { +363 Path rootDir = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir(); +364 Path path = new Path(new Path(rootDir, "export-test"), "export-" + System.currentTimeMillis()); +365 LOG.info("HDFS export destination path: " + path); +366 return path; +367 } +368 +369 private Path getLocalDestinationDir() { +370 Path path = TEST_UTIL.getDataTestDir("local-export-" + System.currentTimeMillis()); +371 LOG.info("Local export destination path: " + path); +372 return path; +373 } +374 +375 private static void removeExportDir(final Path path) throws IOException { +376 FileSystem fs = FileSystem.get(path.toUri(), new Configuration()); +377 fs.delete(path, true); 378 } -379 -380 private static void removeExportDir(final Path path) throws IOException { -381 FileSystem fs = FileSystem.get(path.toUri(), new Configuration()); -382 fs.delete(path, true); -383 } -384} +379} http://git-wip-us.apache.org/repos/asf/hbase-site/blob/6fbeb554/testdevapidocs/src-html/org/apache/hadoop/hbase/snapshot/TestExportSnapshotNoCluster.html ---------------------------------------------------------------------- diff --git a/testdevapidocs/src-html/org/apache/hadoop/hbase/snapshot/TestExportSnapshotNoCluster.html b/testdevapidocs/src-html/org/apache/hadoop/hbase/snapshot/TestExportSnapshotNoCluster.html index 9472257..11cd828 100644 --- a/testdevapidocs/src-html/org/apache/hadoop/hbase/snapshot/TestExportSnapshotNoCluster.html +++ b/testdevapidocs/src-html/org/apache/hadoop/hbase/snapshot/TestExportSnapshotNoCluster.html @@ -26,101 +26,98 @@ 018 019package org.apache.hadoop.hbase.snapshot; 020 -021import static org.junit.Assert.assertEquals; -022import static org.junit.Assert.assertTrue; -023 -024import org.apache.commons.logging.Log; -025import org.apache.commons.logging.LogFactory; -026import org.apache.hadoop.conf.Configuration; -027import org.apache.hadoop.fs.FileSystem; -028import org.apache.hadoop.fs.Path; -029import org.apache.hadoop.hbase.CategoryBasedTimeout; -030import org.apache.hadoop.hbase.HBaseCommonTestingUtility; -031import org.apache.hadoop.hbase.HConstants; -032import org.apache.hadoop.hbase.TableName; -033import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; -034import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils.SnapshotMock; -035import org.apache.hadoop.hbase.testclassification.MediumTests; -036import org.apache.hadoop.hbase.testclassification.MapReduceTests; -037import org.apache.hadoop.hbase.util.Bytes; -038import org.junit.BeforeClass; -039import org.junit.Rule; -040import org.junit.Test; -041import org.junit.experimental.categories.Category; -042import org.junit.rules.TestRule; -043 -044/** -045 * Test Export Snapshot Tool -046 */ -047@Category({MapReduceTests.class, MediumTests.class}) -048public class TestExportSnapshotNoCluster { -049 @Rule public final TestRule timeout = CategoryBasedTimeout.builder(). -050 withTimeout(this.getClass()).withLookingForStuckThread(true).build(); -051 private static final Log LOG = LogFactory.getLog(TestExportSnapshotNoCluster.class); -052 -053 protected final static HBaseCommonTestingUtility TEST_UTIL = new HBaseCommonTestingUtility(); +021import org.apache.commons.logging.Log; +022import org.apache.commons.logging.LogFactory; +023import org.apache.hadoop.conf.Configuration; +024import org.apache.hadoop.fs.FileSystem; +025import org.apache.hadoop.fs.Path; +026import org.apache.hadoop.hbase.CategoryBasedTimeout; +027import org.apache.hadoop.hbase.HBaseCommonTestingUtility; +028import org.apache.hadoop.hbase.HConstants; +029import org.apache.hadoop.hbase.TableName; +030import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; +031import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils.SnapshotMock; +032import org.apache.hadoop.hbase.testclassification.MapReduceTests; +033import org.apache.hadoop.hbase.testclassification.MediumTests; +034import org.apache.hadoop.hbase.util.Bytes; +035import org.junit.BeforeClass; +036import org.junit.Rule; +037import org.junit.Test; +038import org.junit.experimental.categories.Category; +039import org.junit.rules.TestRule; +040 +041/** +042 * Test Export Snapshot Tool +043 */ +044@Category({MapReduceTests.class, MediumTests.class}) +045public class TestExportSnapshotNoCluster { +046 @Rule public final TestRule timeout = CategoryBasedTimeout.builder(). +047 withTimeout(this.getClass()).withLookingForStuckThread(true).build(); +048 private static final Log LOG = LogFactory.getLog(TestExportSnapshotNoCluster.class); +049 +050 protected final static HBaseCommonTestingUtility TEST_UTIL = new HBaseCommonTestingUtility(); +051 +052 private static FileSystem fs; +053 private static Path testDir; 054 -055 private static FileSystem fs; -056 private static Path testDir; -057 -058 public static void setUpBaseConf(Configuration conf) { -059 conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); -060 conf.setInt("hbase.regionserver.msginterval", 100); -061 conf.setInt("hbase.client.pause", 250); -062 conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6); -063 conf.setBoolean("hbase.master.enabletable.roundrobin", true); -064 conf.setInt("mapreduce.map.maxattempts", 10); -065 conf.set(HConstants.HBASE_DIR, testDir.toString()); -066 } -067 -068 @BeforeClass -069 public static void setUpBeforeClass() throws Exception { -070 testDir = TEST_UTIL.getDataTestDir(); -071 fs = testDir.getFileSystem(TEST_UTIL.getConfiguration()); +055 public static void setUpBaseConf(Configuration conf) { +056 conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); +057 conf.setInt("hbase.regionserver.msginterval", 100); +058 conf.setInt("hbase.client.pause", 250); +059 conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6); +060 conf.setBoolean("hbase.master.enabletable.roundrobin", true); +061 conf.setInt("mapreduce.map.maxattempts", 10); +062 conf.set(HConstants.HBASE_DIR, testDir.toString()); +063 } +064 +065 @BeforeClass +066 public static void setUpBeforeClass() throws Exception { +067 testDir = TEST_UTIL.getDataTestDir(); +068 fs = testDir.getFileSystem(TEST_UTIL.getConfiguration()); +069 +070 setUpBaseConf(TEST_UTIL.getConfiguration()); +071 } 072 -073 setUpBaseConf(TEST_UTIL.getConfiguration()); -074 } -075 -076 /** -077 * Mock a snapshot with files in the archive dir, -078 * two regions, and one reference file. -079 */ -080 @Test -081 public void testSnapshotWithRefsExportFileSystemState() throws Exception { -082 SnapshotMock snapshotMock = new SnapshotMock(TEST_UTIL.getConfiguration(), fs, testDir); -083 SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2("tableWithRefsV1", -084 "tableWithRefsV1"); -085 testSnapshotWithRefsExportFileSystemState(builder); -086 -087 snapshotMock = new SnapshotMock(TEST_UTIL.getConfiguration(), fs, testDir); -088 builder = snapshotMock.createSnapshotV2("tableWithRefsV2", "tableWithRefsV2"); -089 testSnapshotWithRefsExportFileSystemState(builder); -090 } -091 -092 /** -093 * Generates a couple of regions for the specified SnapshotMock, -094 * and then it will run the export and verification. -095 */ -096 private void testSnapshotWithRefsExportFileSystemState(SnapshotMock.SnapshotBuilder builder) -097 throws Exception { -098 Path[] r1Files = builder.addRegion(); -099 Path[] r2Files = builder.addRegion(); -100 builder.commit(); -101 int snapshotFilesCount = r1Files.length + r2Files.length; -102 -103 byte[] snapshotName = Bytes.toBytes(builder.getSnapshotDescription().getName()); -104 TableName tableName = builder.getTableDescriptor().getTableName(); -105 TestExportSnapshot.testExportFileSystemState(TEST_UTIL.getConfiguration(), -106 tableName, snapshotName, snapshotName, snapshotFilesCount, -107 testDir, getDestinationDir(), false, null, true); -108 } -109 -110 private Path getDestinationDir() { -111 Path path = new Path(new Path(testDir, "export-test"), "export-" + System.currentTimeMillis()); -112 LOG.info("HDFS export destination path: " + path); -113 return path; -114 } -115} +073 /** +074 * Mock a snapshot with files in the archive dir, +075 * two regions, and one reference file. +076 */ +077 @Test +078 public void testSnapshotWithRefsExportFileSystemState() throws Exception { +079 SnapshotMock snapshotMock = new SnapshotMock(TEST_UTIL.getConfiguration(), fs, testDir); +080 SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2("tableWithRefsV1", +081 "tableWithRefsV1"); +082 testSnapshotWithRefsExportFileSystemState(builder); +083 +084 snapshotMock = new SnapshotMock(TEST_UTIL.getConfiguration(), fs, testDir); +085 builder = snapshotMock.createSnapshotV2("tableWithRefsV2", "tableWithRefsV2"); +086 testSnapshotWithRefsExportFileSystemState(builder); +087 } +088 +089 /** +090 * Generates a couple of regions for the specified SnapshotMock, +091 * and then it will run the export and verification. +092 */ +093 private void testSnapshotWithRefsExportFileSystemState(SnapshotMock.SnapshotBuilder builder) +094 throws Exception { +095 Path[] r1Files = builder.addRegion(); +096 Path[] r2Files = builder.addRegion(); +097 builder.commit(); +098 int snapshotFilesCount = r1Files.length + r2Files.length; +099 +100 byte[] snapshotName = Bytes.toBytes(builder.getSnapshotDescription().getName()); +101 TableName tableName = builder.getTableDescriptor().getTableName(); +102 TestExportSnapshot.testExportFileSystemState(TEST_UTIL.getConfiguration(), +103 tableName, snapshotName, snapshotName, snapshotFilesCount, +104 testDir, getDestinationDir(), false, null, true); +105 } +106 +107 private Path getDestinationDir() { +108 Path path = new Path(new Path(testDir, "export-test"), "export-" + System.currentTimeMillis()); +109 LOG.info("HDFS export destination path: " + path); +110 return path; +111 } +112} http://git-wip-us.apache.org/repos/asf/hbase-site/blob/6fbeb554/testdevapidocs/src-html/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.html ---------------------------------------------------------------------- diff --git a/testdevapidocs/src-html/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.html b/testdevapidocs/src-html/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.html index 748a914..ed6b9e0 100644 --- a/testdevapidocs/src-html/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.html +++ b/testdevapidocs/src-html/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.html @@ -27,55 +27,48 @@ 019package org.apache.hadoop.hbase.snapshot; 020 021 -022import org.apache.commons.logging.Log; -023import org.apache.commons.logging.LogFactory; -024import org.apache.hadoop.conf.Configuration; -025import org.apache.hadoop.hbase.HRegionInfo; -026import org.apache.hadoop.hbase.mob.MobConstants; -027import org.apache.hadoop.hbase.mob.MobUtils; -028import org.apache.hadoop.hbase.testclassification.LargeTests; -029import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests; -030import org.junit.BeforeClass; -031import org.junit.experimental.categories.Category; -032 -033/** -034 * Test Export Snapshot Tool -035 */ -036@Category({VerySlowRegionServerTests.class, LargeTests.class}) -037public class TestMobExportSnapshot extends TestExportSnapshot { -038 private final Log LOG = LogFactory.getLog(getClass()); -039 -040 public static void setUpBaseConf(Configuration conf) { -041 TestExportSnapshot.setUpBaseConf(conf); -042 conf.setInt(MobConstants.MOB_FILE_CACHE_SIZE_KEY, 0); -043 } -044 -045 @BeforeClass -046 public static void setUpBeforeClass() throws Exception { -047 setUpBaseConf(TEST_UTIL.getConfiguration()); -048 // Setup separate test-data directory for MR cluster and set corresponding configurations. -049 // Otherwise, different test classes running MR cluster can step on each other. -050 TEST_UTIL.getDataTestDir(); -051 TEST_UTIL.startMiniZKCluster(); -052 TEST_UTIL.startMiniMapReduceCluster(); -053 TEST_UTIL.startMiniHBaseCluster(1, 3); -054 } -055 -056 @Override -057 protected void createTable() throws Exception { -058 MobSnapshotTestingUtils.createPreSplitMobTable(TEST_UTIL, tableName, 2, FAMILY); -059 } -060 -061 @Override -062 protected RegionPredicate getBypassRegionPredicate() { -063 return new RegionPredicate() { -064 @Override -065 public boolean evaluate(final HRegionInfo regionInfo) { -066 return MobUtils.isMobRegionInfo(regionInfo); -067 } -068 }; -069 } -070} +022import org.apache.hadoop.conf.Configuration; +023import org.apache.hadoop.hbase.HRegionInfo; +024import org.apache.hadoop.hbase.mob.MobConstants; +025import org.apache.hadoop.hbase.mob.MobUtils; +026import org.apache.hadoop.hbase.testclassification.LargeTests; +027import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests; +028import org.junit.BeforeClass; +029import org.junit.experimental.categories.Category; +030 +031/** +032 * Test Export Snapshot Tool +033 */ +034@Category({VerySlowRegionServerTests.class, LargeTests.class}) +035public class TestMobExportSnapshot extends TestExportSnapshot { +036 +037 public static void setUpBaseConf(Configuration conf) { +038 TestExportSnapshot.setUpBaseConf(conf); +039 conf.setInt(MobConstants.MOB_FILE_CACHE_SIZE_KEY, 0); +040 } +041 +042 @BeforeClass +043 public static void setUpBeforeClass() throws Exception { +044 setUpBaseConf(TEST_UTIL.getConfiguration()); +045 TEST_UTIL.startMiniCluster(1, 3); +046 TEST_UTIL.startMiniMapReduceCluster(); +047 } +048 +049 @Override +050 protected void createTable() throws Exception { +051 MobSnapshotTestingUtils.createPreSplitMobTable(TEST_UTIL, tableName, 2, FAMILY); +052 } +053 +054 @Override +055 protected RegionPredicate getBypassRegionPredicate() { +056 return new RegionPredicate() { +057 @Override +058 public boolean evaluate(final HRegionInfo regionInfo) { +059 return MobUtils.isMobRegionInfo(regionInfo); +060 } +061 }; +062 } +063}