Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id DD209200CDC for ; Sun, 2 Jul 2017 01:44:01 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id DC2DE160C06; Sat, 1 Jul 2017 23:44:01 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 703C4160C04 for ; Sun, 2 Jul 2017 01:43:59 +0200 (CEST) Received: (qmail 26559 invoked by uid 500); 1 Jul 2017 23:43:57 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 26231 invoked by uid 99); 1 Jul 2017 23:43:57 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 01 Jul 2017 23:43:57 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 3AC7FF2178; Sat, 1 Jul 2017 23:43:54 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: git-site-role@apache.org To: commits@hbase.apache.org Date: Sat, 01 Jul 2017 23:43:56 -0000 Message-Id: <7a31b71cd3be4e618f6c1108660cb748@git.apache.org> In-Reply-To: <7acdac7676004726a47bdcef7bac7d48@git.apache.org> References: <7acdac7676004726a47bdcef7bac7d48@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [03/21] hbase-site git commit: Published site at 82d554e3783372cc6b05489452c815b57c06f6cd. archived-at: Sat, 01 Jul 2017 23:44:02 -0000 http://git-wip-us.apache.org/repos/asf/hbase-site/blob/640333c3/testdevapidocs/src-html/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.SnapshotMock.SnapshotBuilder.html ---------------------------------------------------------------------- diff --git a/testdevapidocs/src-html/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.SnapshotMock.SnapshotBuilder.html b/testdevapidocs/src-html/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.SnapshotMock.SnapshotBuilder.html index be69921..f7f103c 100644 --- a/testdevapidocs/src-html/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.SnapshotMock.SnapshotBuilder.html +++ b/testdevapidocs/src-html/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.SnapshotMock.SnapshotBuilder.html @@ -217,7 +217,7 @@ 209 // check snapshot dir 210 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir( 211 snapshotDescriptor, rootDir); -212 assertTrue(fs.exists(snapshotDir)); +212 assertTrue("target snapshot directory, '"+ snapshotDir +"', doesn't exist.", fs.exists(snapshotDir)); 213 214 SnapshotProtos.SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); 215 @@ -240,14 +240,14 @@ 232 // Verify that there are store files in the specified families 233 if (nonEmptyTestFamilies != null) { 234 for (final byte[] familyName: nonEmptyTestFamilies) { -235 assertTrue(snapshotFamilies.contains(familyName)); +235 assertTrue("Expected snapshot to contain family '" + Bytes.toString(familyName) + "', but it does not.", snapshotFamilies.contains(familyName)); 236 } 237 } 238 239 // Verify that there are no store files in the specified families 240 if (emptyTestFamilies != null) { 241 for (final byte[] familyName: emptyTestFamilies) { -242 assertFalse(snapshotFamilies.contains(familyName)); +242 assertFalse("Expected snapshot to skip empty family '" + Bytes.toString(familyName) + "', but it is present.", snapshotFamilies.contains(familyName)); 243 } 244 } 245 @@ -258,7 +258,7 @@ 250 boolean hasMob = regionManifests.containsKey(MobUtils.getMobRegionInfo(tableName) 251 .getEncodedName()); 252 if (hasMob) { -253 assertEquals(regions.size(), regionManifests.size() - 1); +253 assertEquals("Wrong number of regions.", regions.size(), regionManifests.size() - 1); 254 } else { 255 // if create snapshot when table splitting, parent region will be included to the snapshot 256 // region manifest. we should exclude the parent regions. @@ -270,13 +270,13 @@ 262 } 263 regionCountExclusiveSplitParent++; 264 } -265 assertEquals(regions.size(), regionCountExclusiveSplitParent); +265 assertEquals("Wrong number of regions.", regions.size(), regionCountExclusiveSplitParent); 266 } 267 268 // Verify Regions (redundant check, see MasterSnapshotVerifier) 269 for (HRegionInfo info : regions) { 270 String regionName = info.getEncodedName(); -271 assertTrue(regionManifests.containsKey(regionName)); +271 assertTrue("Missing region name: '" + regionName + "'", regionManifests.containsKey(regionName)); 272 } 273 } 274 @@ -411,496 +411,500 @@ 403 throws Exception { 404 if (!onlineSnapshot) { 405 try { -406 admin.disableTable(tableName); -407 } catch (TableNotEnabledException tne) { -408 LOG.info("In attempting to disable " + tableName + " it turns out that the this table is " + -409 "already disabled."); -410 } -411 } -412 admin.snapshot(snapshotNameString, tableName); -413 -414 List<SnapshotDescription> snapshots = -415 SnapshotTestingUtils.assertExistsMatchingSnapshot(admin, snapshotNameString, tableName); -416 if (snapshots == null || snapshots.size() != 1) { -417 Assert.fail("Incorrect number of snapshots for table " + tableName); -418 } -419 -420 SnapshotTestingUtils.confirmSnapshotValid( -421 ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), tableName, nonEmptyFamilyNames, -422 emptyFamilyNames, rootDir, admin, fs); -423 } -424 -425 /** -426 * Corrupt the specified snapshot by deleting some files. -427 * -428 * @param util {@link HBaseTestingUtility} -429 * @param snapshotName name of the snapshot to corrupt -430 * @return array of the corrupted HFiles -431 * @throws IOException on unexecpted error reading the FS -432 */ -433 public static ArrayList corruptSnapshot(final HBaseTestingUtility util, final String snapshotName) -434 throws IOException { -435 final MasterFileSystem mfs = util.getHBaseCluster().getMaster().getMasterFileSystem(); -436 final FileSystem fs = mfs.getFileSystem(); -437 -438 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, -439 mfs.getRootDir()); -440 SnapshotProtos.SnapshotDescription snapshotDesc = -441 SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); -442 final TableName table = TableName.valueOf(snapshotDesc.getTable()); -443 -444 final ArrayList corruptedFiles = new ArrayList(); -445 final Configuration conf = util.getConfiguration(); -446 SnapshotReferenceUtil.visitTableStoreFiles(conf, fs, snapshotDir, snapshotDesc, -447 new SnapshotReferenceUtil.StoreFileVisitor() { -448 @Override -449 public void storeFile(final HRegionInfo regionInfo, final String family, -450 final SnapshotRegionManifest.StoreFile storeFile) throws IOException { -451 String region = regionInfo.getEncodedName(); -452 String hfile = storeFile.getName(); -453 HFileLink link = HFileLink.build(conf, table, region, family, hfile); -454 if (corruptedFiles.size() % 2 == 0) { -455 fs.delete(link.getAvailablePath(fs), true); -456 corruptedFiles.add(hfile); -457 } -458 } -459 }); -460 -461 assertTrue(corruptedFiles.size() > 0); -462 return corruptedFiles; -463 } +406 LOG.info("prepping for offline snapshot."); +407 admin.disableTable(tableName); +408 } catch (TableNotEnabledException tne) { +409 LOG.info("In attempting to disable " + tableName + " it turns out that the this table is " + +410 "already disabled."); +411 } +412 } +413 LOG.info("taking snapshot."); +414 admin.snapshot(snapshotNameString, tableName); +415 +416 LOG.info("Confirming snapshot exists."); +417 List<SnapshotDescription> snapshots = +418 SnapshotTestingUtils.assertExistsMatchingSnapshot(admin, snapshotNameString, tableName); +419 if (snapshots == null || snapshots.size() != 1) { +420 Assert.fail("Incorrect number of snapshots for table " + tableName); +421 } +422 +423 LOG.info("validating snapshot."); +424 SnapshotTestingUtils.confirmSnapshotValid( +425 ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), tableName, nonEmptyFamilyNames, +426 emptyFamilyNames, rootDir, admin, fs); +427 } +428 +429 /** +430 * Corrupt the specified snapshot by deleting some files. +431 * +432 * @param util {@link HBaseTestingUtility} +433 * @param snapshotName name of the snapshot to corrupt +434 * @return array of the corrupted HFiles +435 * @throws IOException on unexecpted error reading the FS +436 */ +437 public static ArrayList corruptSnapshot(final HBaseTestingUtility util, final String snapshotName) +438 throws IOException { +439 final MasterFileSystem mfs = util.getHBaseCluster().getMaster().getMasterFileSystem(); +440 final FileSystem fs = mfs.getFileSystem(); +441 +442 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, +443 mfs.getRootDir()); +444 SnapshotProtos.SnapshotDescription snapshotDesc = +445 SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); +446 final TableName table = TableName.valueOf(snapshotDesc.getTable()); +447 +448 final ArrayList corruptedFiles = new ArrayList(); +449 final Configuration conf = util.getConfiguration(); +450 SnapshotReferenceUtil.visitTableStoreFiles(conf, fs, snapshotDir, snapshotDesc, +451 new SnapshotReferenceUtil.StoreFileVisitor() { +452 @Override +453 public void storeFile(final HRegionInfo regionInfo, final String family, +454 final SnapshotRegionManifest.StoreFile storeFile) throws IOException { +455 String region = regionInfo.getEncodedName(); +456 String hfile = storeFile.getName(); +457 HFileLink link = HFileLink.build(conf, table, region, family, hfile); +458 if (corruptedFiles.size() % 2 == 0) { +459 fs.delete(link.getAvailablePath(fs), true); +460 corruptedFiles.add(hfile); +461 } +462 } +463 }); 464 -465 // ========================================================================== -466 // Snapshot Mock -467 // ========================================================================== -468 public static class SnapshotMock { -469 protected final static String TEST_FAMILY = "cf"; -470 public final static int TEST_NUM_REGIONS = 4; -471 -472 private final Configuration conf; -473 private final FileSystem fs; -474 private final Path rootDir; +465 assertTrue(corruptedFiles.size() > 0); +466 return corruptedFiles; +467 } +468 +469 // ========================================================================== +470 // Snapshot Mock +471 // ========================================================================== +472 public static class SnapshotMock { +473 protected final static String TEST_FAMILY = "cf"; +474 public final static int TEST_NUM_REGIONS = 4; 475 -476 static class RegionData { -477 public HRegionInfo hri; -478 public Path tableDir; -479 public Path[] files; -480 -481 public RegionData(final Path tableDir, final HRegionInfo hri, final int nfiles) { -482 this.tableDir = tableDir; -483 this.hri = hri; -484 this.files = new Path[nfiles]; -485 } -486 } -487 -488 public static class SnapshotBuilder { -489 private final RegionData[] tableRegions; -490 private final SnapshotProtos.SnapshotDescription desc; -491 private final HTableDescriptor htd; -492 private final Configuration conf; -493 private final FileSystem fs; -494 private final Path rootDir; -495 private Path snapshotDir; -496 private int snapshotted = 0; -497 -498 public SnapshotBuilder(final Configuration conf, final FileSystem fs, -499 final Path rootDir, final HTableDescriptor htd, -500 final SnapshotProtos.SnapshotDescription desc, final RegionData[] tableRegions) -501 throws IOException { -502 this.fs = fs; -503 this.conf = conf; -504 this.rootDir = rootDir; -505 this.htd = htd; -506 this.desc = desc; -507 this.tableRegions = tableRegions; -508 this.snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir); -509 new FSTableDescriptors(conf) -510 .createTableDescriptorForTableDirectory(snapshotDir, htd, false); -511 } -512 -513 public HTableDescriptor getTableDescriptor() { -514 return this.htd; +476 private final Configuration conf; +477 private final FileSystem fs; +478 private final Path rootDir; +479 +480 static class RegionData { +481 public HRegionInfo hri; +482 public Path tableDir; +483 public Path[] files; +484 +485 public RegionData(final Path tableDir, final HRegionInfo hri, final int nfiles) { +486 this.tableDir = tableDir; +487 this.hri = hri; +488 this.files = new Path[nfiles]; +489 } +490 } +491 +492 public static class SnapshotBuilder { +493 private final RegionData[] tableRegions; +494 private final SnapshotProtos.SnapshotDescription desc; +495 private final HTableDescriptor htd; +496 private final Configuration conf; +497 private final FileSystem fs; +498 private final Path rootDir; +499 private Path snapshotDir; +500 private int snapshotted = 0; +501 +502 public SnapshotBuilder(final Configuration conf, final FileSystem fs, +503 final Path rootDir, final HTableDescriptor htd, +504 final SnapshotProtos.SnapshotDescription desc, final RegionData[] tableRegions) +505 throws IOException { +506 this.fs = fs; +507 this.conf = conf; +508 this.rootDir = rootDir; +509 this.htd = htd; +510 this.desc = desc; +511 this.tableRegions = tableRegions; +512 this.snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir); +513 new FSTableDescriptors(conf) +514 .createTableDescriptorForTableDirectory(snapshotDir, htd, false); 515 } 516 -517 public SnapshotProtos.SnapshotDescription getSnapshotDescription() { -518 return this.desc; +517 public HTableDescriptor getTableDescriptor() { +518 return this.htd; 519 } 520 -521 public Path getSnapshotsDir() { -522 return this.snapshotDir; +521 public SnapshotProtos.SnapshotDescription getSnapshotDescription() { +522 return this.desc; 523 } 524 -525 public Path[] addRegion() throws IOException { -526 return addRegion(desc); +525 public Path getSnapshotsDir() { +526 return this.snapshotDir; 527 } 528 -529 public Path[] addRegionV1() throws IOException { -530 return addRegion(desc.toBuilder() -531 .setVersion(SnapshotManifestV1.DESCRIPTOR_VERSION) -532 .build()); -533 } -534 -535 public Path[] addRegionV2() throws IOException { -536 return addRegion(desc.toBuilder() -537 .setVersion(SnapshotManifestV2.DESCRIPTOR_VERSION) -538 .build()); -539 } -540 -541 private Path[] addRegion(final SnapshotProtos.SnapshotDescription desc) throws IOException { -542 if (this.snapshotted == tableRegions.length) { -543 throw new UnsupportedOperationException("No more regions in the table"); -544 } -545 -546 RegionData regionData = tableRegions[this.snapshotted++]; -547 ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName()); -548 SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor); -549 manifest.addRegion(regionData.tableDir, regionData.hri); -550 return regionData.files; -551 } -552 -553 private void corruptFile(Path p) throws IOException { -554 String manifestName = p.getName(); -555 -556 // Rename the original region-manifest file -557 Path newP = new Path(p.getParent(), manifestName + "1"); -558 fs.rename(p, newP); +529 public Path[] addRegion() throws IOException { +530 return addRegion(desc); +531 } +532 +533 public Path[] addRegionV1() throws IOException { +534 return addRegion(desc.toBuilder() +535 .setVersion(SnapshotManifestV1.DESCRIPTOR_VERSION) +536 .build()); +537 } +538 +539 public Path[] addRegionV2() throws IOException { +540 return addRegion(desc.toBuilder() +541 .setVersion(SnapshotManifestV2.DESCRIPTOR_VERSION) +542 .build()); +543 } +544 +545 private Path[] addRegion(final SnapshotProtos.SnapshotDescription desc) throws IOException { +546 if (this.snapshotted == tableRegions.length) { +547 throw new UnsupportedOperationException("No more regions in the table"); +548 } +549 +550 RegionData regionData = tableRegions[this.snapshotted++]; +551 ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName()); +552 SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor); +553 manifest.addRegion(regionData.tableDir, regionData.hri); +554 return regionData.files; +555 } +556 +557 private void corruptFile(Path p) throws IOException { +558 String manifestName = p.getName(); 559 -560 // Create a new region-manifest file -561 FSDataOutputStream out = fs.create(p); -562 -563 //Copy the first 25 bytes of the original region-manifest into the new one, -564 //make it a corrupted region-manifest file. -565 FSDataInputStream input = fs.open(newP); -566 byte[] buffer = new byte[25]; -567 int len = input.read(0, buffer, 0, 25); -568 if (len > 1) { -569 out.write(buffer, 0, len - 1); -570 } -571 out.close(); -572 -573 // Delete the original region-manifest -574 fs.delete(newP); -575 } +560 // Rename the original region-manifest file +561 Path newP = new Path(p.getParent(), manifestName + "1"); +562 fs.rename(p, newP); +563 +564 // Create a new region-manifest file +565 FSDataOutputStream out = fs.create(p); +566 +567 //Copy the first 25 bytes of the original region-manifest into the new one, +568 //make it a corrupted region-manifest file. +569 FSDataInputStream input = fs.open(newP); +570 byte[] buffer = new byte[25]; +571 int len = input.read(0, buffer, 0, 25); +572 if (len > 1) { +573 out.write(buffer, 0, len - 1); +574 } +575 out.close(); 576 -577 /** -578 * Corrupt one region-manifest file -579 * -580 * @throws IOException on unexecpted error from the FS -581 */ -582 public void corruptOneRegionManifest() throws IOException { -583 FileStatus[] manifestFiles = FSUtils.listStatus(fs, snapshotDir, new PathFilter() { -584 @Override public boolean accept(Path path) { -585 return path.getName().startsWith(SnapshotManifestV2.SNAPSHOT_MANIFEST_PREFIX); -586 } -587 }); -588 -589 if (manifestFiles.length == 0) return; -590 -591 // Just choose the first one -592 Path p = manifestFiles[0].getPath(); -593 corruptFile(p); -594 } -595 -596 public void missOneRegionSnapshotFile() throws IOException { -597 FileStatus[] manifestFiles = FSUtils.listStatus(fs, snapshotDir); -598 for (FileStatus fileStatus : manifestFiles) { -599 String fileName = fileStatus.getPath().getName(); -600 if (fileName.endsWith(SnapshotDescriptionUtils.SNAPSHOTINFO_FILE) -601 || fileName.endsWith(".tabledesc") -602 || fileName.endsWith(SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME)) { -603 fs.delete(fileStatus.getPath(), true); -604 } -605 } -606 } -607 -608 /** -609 * Corrupt data-manifest file -610 * -611 * @throws IOException on unexecpted error from the FS -612 */ -613 public void corruptDataManifest() throws IOException { -614 FileStatus[] manifestFiles = FSUtils.listStatus(fs, snapshotDir, new PathFilter() { -615 @Override -616 public boolean accept(Path path) { -617 return path.getName().startsWith(SnapshotManifest.DATA_MANIFEST_NAME); -618 } -619 }); -620 -621 if (manifestFiles.length == 0) return; -622 -623 // Just choose the first one -624 Path p = manifestFiles[0].getPath(); -625 corruptFile(p); -626 } -627 -628 public Path commit() throws IOException { -629 ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName()); -630 SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor); -631 manifest.addTableDescriptor(htd); -632 manifest.consolidate(); -633 SnapshotDescriptionUtils.completeSnapshot(desc, rootDir, snapshotDir, fs); -634 snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(desc, rootDir); -635 return snapshotDir; -636 } -637 -638 public void consolidate() throws IOException { -639 ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName()); -640 SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor); -641 manifest.addTableDescriptor(htd); -642 manifest.consolidate(); -643 } -644 } -645 -646 public SnapshotMock(final Configuration conf, final FileSystem fs, final Path rootDir) { -647 this.fs = fs; -648 this.conf = conf; -649 this.rootDir = rootDir; -650 } -651 -652 public SnapshotBuilder createSnapshotV1(final String snapshotName, final String tableName) -653 throws IOException { -654 return createSnapshot(snapshotName, tableName, SnapshotManifestV1.DESCRIPTOR_VERSION); -655 } -656 -657 public SnapshotBuilder createSnapshotV1(final String snapshotName, final String tableName, -658 final int numRegions) throws IOException { -659 return createSnapshot(snapshotName, tableName, numRegions, SnapshotManifestV1.DESCRIPTOR_VERSION); -660 } -661 -662 public SnapshotBuilder createSnapshotV2(final String snapshotName, final String tableName) -663 throws IOException { -664 return createSnapshot(snapshotName, tableName, SnapshotManifestV2.DESCRIPTOR_VERSION); -665 } -666 -667 public SnapshotBuilder createSnapshotV2(final String snapshotName, final String tableName, -668 final int numRegions) throws IOException { -669 return createSnapshot(snapshotName, tableName, numRegions, SnapshotManifestV2.DESCRIPTOR_VERSION); -670 } -671 -672 private SnapshotBuilder createSnapshot(final String snapshotName, final String tableName, -673 final int version) throws IOException { -674 return createSnapshot(snapshotName, tableName, TEST_NUM_REGIONS, version); -675 } -676 -677 private SnapshotBuilder createSnapshot(final String snapshotName, final String tableName, -678 final int numRegions, final int version) throws IOException { -679 HTableDescriptor htd = createHtd(tableName); -680 RegionData[] regions = createTable(htd, numRegions); -681 -682 SnapshotProtos.SnapshotDescription desc = SnapshotProtos.SnapshotDescription.newBuilder() -683 .setTable(htd.getNameAsString()) -684 .setName(snapshotName) -685 .setVersion(version) -686 .build(); -687 -688 Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir); -689 SnapshotDescriptionUtils.writeSnapshotInfo(desc, workingDir, fs); -690 return new SnapshotBuilder(conf, fs, rootDir, htd, desc, regions); -691 } -692 -693 public HTableDescriptor createHtd(final String tableName) { -694 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName)); -695 htd.addFamily(new HColumnDescriptor(TEST_FAMILY)); -696 return htd; -697 } -698 -699 private RegionData[] createTable(final HTableDescriptor htd, final int nregions) -700 throws IOException { -701 Path tableDir = FSUtils.getTableDir(rootDir, htd.getTableName()); -702 new FSTableDescriptors(conf).createTableDescriptorForTableDirectory(tableDir, htd, false); -703 -704 assertTrue(nregions % 2 == 0); -705 RegionData[] regions = new RegionData[nregions]; -706 for (int i = 0; i < regions.length; i += 2) { -707 byte[] startKey = Bytes.toBytes(0 + i * 2); -708 byte[] endKey = Bytes.toBytes(1 + i * 2); -709 -710 // First region, simple with one plain hfile. -711 HRegionInfo hri = new HRegionInfo(htd.getTableName(), startKey, endKey); -712 HRegionFileSystem rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri); -713 regions[i] = new RegionData(tableDir, hri, 3); -714 for (int j = 0; j < regions[i].files.length; ++j) { -715 Path storeFile = createStoreFile(rfs.createTempName()); -716 regions[i].files[j] = rfs.commitStoreFile(TEST_FAMILY, storeFile); -717 } -718 -719 // Second region, used to test the split case. -720 // This region contains a reference to the hfile in the first region. -721 startKey = Bytes.toBytes(2 + i * 2); -722 endKey = Bytes.toBytes(3 + i * 2); -723 hri = new HRegionInfo(htd.getTableName()); -724 rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri); -725 regions[i+1] = new RegionData(tableDir, hri, regions[i].files.length); -726 for (int j = 0; j < regions[i].files.length; ++j) { -727 String refName = regions[i].files[j].getName() + '.' + regions[i].hri.getEncodedName(); -728 Path refFile = createStoreFile(new Path(rootDir, refName)); -729 regions[i+1].files[j] = rfs.commitStoreFile(TEST_FAMILY, refFile); -730 } -731 } -732 return regions; -733 } -734 -735 private Path createStoreFile(final Path storeFile) -736 throws IOException { -737 FSDataOutputStream out = fs.create(storeFile); -738 try { -739 out.write(Bytes.toBytes(storeFile.toString())); -740 } finally { -741 out.close(); -742 } -743 return storeFile; -744 } -745 } -746 -747 // ========================================================================== -748 // Table Helpers -749 // ========================================================================== -750 public static void waitForTableToBeOnline(final HBaseTestingUtility util, -751 final TableName tableName) -752 throws IOException, InterruptedException { -753 HRegionServer rs = util.getRSForFirstRegionInTable(tableName); -754 List<Region> onlineRegions = rs.getOnlineRegions(tableName); -755 for (Region region : onlineRegions) { -756 region.waitForFlushesAndCompactions(); -757 } -758 // Wait up to 60 seconds for a table to be available. -759 util.waitFor(60000, util.predicateTableAvailable(tableName)); -760 } -761 -762 public static void createTable(final HBaseTestingUtility util, final TableName tableName, -763 int regionReplication, int nRegions, final byte[]... families) -764 throws IOException, InterruptedException { -765 HTableDescriptor htd = new HTableDescriptor(tableName); -766 htd.setRegionReplication(regionReplication); -767 for (byte[] family : families) { -768 HColumnDescriptor hcd = new HColumnDescriptor(family); -769 htd.addFamily(hcd); -770 } -771 byte[][] splitKeys = getSplitKeys(nRegions); -772 util.createTable(htd, splitKeys); -773 assertEquals((splitKeys.length + 1) * regionReplication, -774 util.getAdmin().getTableRegions(tableName).size()); -775 } -776 -777 public static byte[][] getSplitKeys() { -778 return getSplitKeys(KEYS.length); +577 // Delete the original region-manifest +578 fs.delete(newP); +579 } +580 +581 /** +582 * Corrupt one region-manifest file +583 * +584 * @throws IOException on unexecpted error from the FS +585 */ +586 public void corruptOneRegionManifest() throws IOException { +587 FileStatus[] manifestFiles = FSUtils.listStatus(fs, snapshotDir, new PathFilter() { +588 @Override public boolean accept(Path path) { +589 return path.getName().startsWith(SnapshotManifestV2.SNAPSHOT_MANIFEST_PREFIX); +590 } +591 }); +592 +593 if (manifestFiles.length == 0) return; +594 +595 // Just choose the first one +596 Path p = manifestFiles[0].getPath(); +597 corruptFile(p); +598 } +599 +600 public void missOneRegionSnapshotFile() throws IOException { +601 FileStatus[] manifestFiles = FSUtils.listStatus(fs, snapshotDir); +602 for (FileStatus fileStatus : manifestFiles) { +603 String fileName = fileStatus.getPath().getName(); +604 if (fileName.endsWith(SnapshotDescriptionUtils.SNAPSHOTINFO_FILE) +605 || fileName.endsWith(".tabledesc") +606 || fileName.endsWith(SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME)) { +607 fs.delete(fileStatus.getPath(), true); +608 } +609 } +610 } +611 +612 /** +613 * Corrupt data-manifest file +614 * +615 * @throws IOException on unexecpted error from the FS +616 */ +617 public void corruptDataManifest() throws IOException { +618 FileStatus[] manifestFiles = FSUtils.listStatus(fs, snapshotDir, new PathFilter() { +619 @Override +620 public boolean accept(Path path) { +621 return path.getName().startsWith(SnapshotManifest.DATA_MANIFEST_NAME); +622 } +623 }); +624 +625 if (manifestFiles.length == 0) return; +626 +627 // Just choose the first one +628 Path p = manifestFiles[0].getPath(); +629 corruptFile(p); +630 } +631 +632 public Path commit() throws IOException { +633 ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName()); +634 SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor); +635 manifest.addTableDescriptor(htd); +636 manifest.consolidate(); +637 SnapshotDescriptionUtils.completeSnapshot(desc, rootDir, snapshotDir, fs); +638 snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(desc, rootDir); +639 return snapshotDir; +640 } +641 +642 public void consolidate() throws IOException { +643 ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName()); +644 SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor); +645 manifest.addTableDescriptor(htd); +646 manifest.consolidate(); +647 } +648 } +649 +650 public SnapshotMock(final Configuration conf, final FileSystem fs, final Path rootDir) { +651 this.fs = fs; +652 this.conf = conf; +653 this.rootDir = rootDir; +654 } +655 +656 public SnapshotBuilder createSnapshotV1(final String snapshotName, final String tableName) +657 throws IOException { +658 return createSnapshot(snapshotName, tableName, SnapshotManifestV1.DESCRIPTOR_VERSION); +659 } +660 +661 public SnapshotBuilder createSnapshotV1(final String snapshotName, final String tableName, +662 final int numRegions) throws IOException { +663 return createSnapshot(snapshotName, tableName, numRegions, SnapshotManifestV1.DESCRIPTOR_VERSION); +664 } +665 +666 public SnapshotBuilder createSnapshotV2(final String snapshotName, final String tableName) +667 throws IOException { +668 return createSnapshot(snapshotName, tableName, SnapshotManifestV2.DESCRIPTOR_VERSION); +669 } +670 +671 public SnapshotBuilder createSnapshotV2(final String snapshotName, final String tableName, +672 final int numRegions) throws IOException { +673 return createSnapshot(snapshotName, tableName, numRegions, SnapshotManifestV2.DESCRIPTOR_VERSION); +674 } +675 +676 private SnapshotBuilder createSnapshot(final String snapshotName, final String tableName, +677 final int version) throws IOException { +678 return createSnapshot(snapshotName, tableName, TEST_NUM_REGIONS, version); +679 } +680 +681 private SnapshotBuilder createSnapshot(final String snapshotName, final String tableName, +682 final int numRegions, final int version) throws IOException { +683 HTableDescriptor htd = createHtd(tableName); +684 RegionData[] regions = createTable(htd, numRegions); +685 +686 SnapshotProtos.SnapshotDescription desc = SnapshotProtos.SnapshotDescription.newBuilder() +687 .setTable(htd.getNameAsString()) +688 .setName(snapshotName) +689 .setVersion(version) +690 .build(); +691 +692 Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir); +693 SnapshotDescriptionUtils.writeSnapshotInfo(desc, workingDir, fs); +694 return new SnapshotBuilder(conf, fs, rootDir, htd, desc, regions); +695 } +696 +697 public HTableDescriptor createHtd(final String tableName) { +698 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName)); +699 htd.addFamily(new HColumnDescriptor(TEST_FAMILY)); +700 return htd; +701 } +702 +703 private RegionData[] createTable(final HTableDescriptor htd, final int nregions) +704 throws IOException { +705 Path tableDir = FSUtils.getTableDir(rootDir, htd.getTableName()); +706 new FSTableDescriptors(conf).createTableDescriptorForTableDirectory(tableDir, htd, false); +707 +708 assertTrue(nregions % 2 == 0); +709 RegionData[] regions = new RegionData[nregions]; +710 for (int i = 0; i < regions.length; i += 2) { +711 byte[] startKey = Bytes.toBytes(0 + i * 2); +712 byte[] endKey = Bytes.toBytes(1 + i * 2); +713 +714 // First region, simple with one plain hfile. +715 HRegionInfo hri = new HRegionInfo(htd.getTableName(), startKey, endKey); +716 HRegionFileSystem rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri); +717 regions[i] = new RegionData(tableDir, hri, 3); +718 for (int j = 0; j < regions[i].files.length; ++j) { +719 Path storeFile = createStoreFile(rfs.createTempName()); +720 regions[i].files[j] = rfs.commitStoreFile(TEST_FAMILY, storeFile); +721 } +722 +723 // Second region, used to test the split case. +724 // This region contains a reference to the hfile in the first region. +725 startKey = Bytes.toBytes(2 + i * 2); +726 endKey = Bytes.toBytes(3 + i * 2); +727 hri = new HRegionInfo(htd.getTableName()); +728 rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri); +729 regions[i+1] = new RegionData(tableDir, hri, regions[i].files.length); +730 for (int j = 0; j < regions[i].files.length; ++j) { +731 String refName = regions[i].files[j].getName() + '.' + regions[i].hri.getEncodedName(); +732 Path refFile = createStoreFile(new Path(rootDir, refName)); +733 regions[i+1].files[j] = rfs.commitStoreFile(TEST_FAMILY, refFile); +734 } +735 } +736 return regions; +737 } +738 +739 private Path createStoreFile(final Path storeFile) +740 throws IOException { +741 FSDataOutputStream out = fs.create(storeFile); +742 try { +743 out.write(Bytes.toBytes(storeFile.toString())); +744 } finally { +745 out.close(); +746 } +747 return storeFile; +748 } +749 } +750 +751 // ========================================================================== +752 // Table Helpers +753 // ========================================================================== +754 public static void waitForTableToBeOnline(final HBaseTestingUtility util, +755 final TableName tableName) +756 throws IOException, InterruptedException { +757 HRegionServer rs = util.getRSForFirstRegionInTable(tableName); +758 List<Region> onlineRegions = rs.getOnlineRegions(tableName); +759 for (Region region : onlineRegions) { +760 region.waitForFlushesAndCompactions(); +761 } +762 // Wait up to 60 seconds for a table to be available. +763 util.waitFor(60000, util.predicateTableAvailable(tableName)); +764 } +765 +766 public static void createTable(final HBaseTestingUtility util, final TableName tableName, +767 int regionReplication, int nRegions, final byte[]... families) +768 throws IOException, InterruptedException { +769 HTableDescriptor htd = new HTableDescriptor(tableName); +770 htd.setRegionReplication(regionReplication); +771 for (byte[] family : families) { +772 HColumnDescriptor hcd = new HColumnDescriptor(family); +773 htd.addFamily(hcd); +774 } +775 byte[][] splitKeys = getSplitKeys(nRegions); +776 util.createTable(htd, splitKeys); +777 assertEquals((splitKeys.length + 1) * regionReplication, +778 util.getAdmin().getTableRegions(tableName).size()); 779 } 780 -781 public static byte[][] getSplitKeys(int nRegions) { -782 nRegions = nRegions < KEYS.length ? nRegions : (KEYS.length - 1); -783 final byte[][] splitKeys = new byte[nRegions-1][]; -784 final int step = KEYS.length / nRegions; -785 int keyIndex = 1; -786 for (int i = 0; i < splitKeys.length; ++i) { -787 splitKeys[i] = new byte[] { KEYS[keyIndex] }; -788 keyIndex += step; -789 } -790 return splitKeys; -791 } -792 -793 public static void createTable(final HBaseTestingUtility util, final TableName tableName, -794 final byte[]... families) throws IOException, InterruptedException { -795 createTable(util, tableName, 1, families); -796 } -797 -798 public static void createTable(final HBaseTestingUtility util, final TableName tableName, -799 final int regionReplication, final byte[]... families) throws IOException, InterruptedException { -800 createTable(util, tableName, regionReplication, KEYS.length, families); -801 } -802 -803 public static void createPreSplitTable(final HBaseTestingUtility util, final TableName tableName, -804 final int nRegions, final byte[]... families) throws IOException, InterruptedException { -805 createTable(util, tableName, 1, nRegions, families); -806 } -807 -808 public static void loadData(final HBaseTestingUtility util, final TableName tableName, int rows, -809 byte[]... families) throws IOException, InterruptedException { -810 BufferedMutator mutator = util.getConnection().getBufferedMutator(tableName); -811 loadData(util, mutator, rows, families); -812 } -813 -814 public static void loadData(final HBaseTestingUtility util, final BufferedMutator mutator, int rows, -815 byte[]... families) throws IOException, InterruptedException { -816 // Ensure one row per region -817 assertTrue(rows >= KEYS.length); -818 for (byte k0: KEYS) { -819 byte[] k = new byte[] { k0 }; -820 byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), k); -821 byte[] key = Bytes.add(k, Bytes.toBytes(MD5Hash.getMD5AsHex(value))); -822 final byte[][] families1 = families; -823 final byte[] key1 = key; -824 final byte[] value1 = value; -825 mutator.mutate(createPut(families1, key1, value1)); -826 rows--; -827 } -828 -829 // Add other extra rows. more rows, more files -830 while (rows-- > 0) { -831 byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), Bytes.toBytes(rows)); -832 byte[] key = Bytes.toBytes(MD5Hash.getMD5AsHex(value)); -833 final byte[][] families1 = families; -834 final byte[] key1 = key; -835 final byte[] value1 = value; -836 mutator.mutate(createPut(families1, key1, value1)); -837 } -838 mutator.flush(); -839 -840 waitForTableToBeOnline(util, mutator.getName()); -841 } -842 -843 private static Put createPut(final byte[][] families, final byte[] key, final byte[] value) { -844 byte[] q = Bytes.toBytes("q"); -845 Put put = new Put(key); -846 put.setDurability(Durability.SKIP_WAL); -847 for (byte[] family: families) { -848 put.addColumn(family, q, value); -849 } -850 return put; -851 } -852 -853 public static void deleteAllSnapshots(final Admin admin) -854 throws IOException { -855 // Delete all the snapshots -856 for (SnapshotDescription snapshot: admin.listSnapshots()) { -857 admin.deleteSnapshot(snapshot.getName()); -858 } -859 SnapshotTestingUtils.assertNoSnapshots(admin); -860 } -861 -862 public static void deleteArchiveDirectory(final HBaseTesting