Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 3F2F8200CA5 for ; Fri, 26 May 2017 16:59:56 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 3DE85160BD9; Fri, 26 May 2017 14:59:56 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id E7AFA160BDC for ; Fri, 26 May 2017 16:59:53 +0200 (CEST) Received: (qmail 41159 invoked by uid 500); 26 May 2017 14:59:52 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 40334 invoked by uid 99); 26 May 2017 14:59:52 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 26 May 2017 14:59:52 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 1294CE04F2; Fri, 26 May 2017 14:59:52 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: git-site-role@apache.org To: commits@hbase.apache.org Date: Fri, 26 May 2017 14:59:54 -0000 Message-Id: <2c0f997b155f4d1fb783444f3ca2e33e@git.apache.org> In-Reply-To: <08535aace7c44c53b36808b45cf62c2c@git.apache.org> References: <08535aace7c44c53b36808b45cf62c2c@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [04/32] hbase-site git commit: Published site at 82d554e3783372cc6b05489452c815b57c06f6cd. archived-at: Fri, 26 May 2017 14:59:56 -0000 http://git-wip-us.apache.org/repos/asf/hbase-site/blob/be6f38d2/testdevapidocs/src-html/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.SnapshotMock.RegionData.html ---------------------------------------------------------------------- diff --git a/testdevapidocs/src-html/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.SnapshotMock.RegionData.html b/testdevapidocs/src-html/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.SnapshotMock.RegionData.html index c07cc8b..be69921 100644 --- a/testdevapidocs/src-html/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.SnapshotMock.RegionData.html +++ b/testdevapidocs/src-html/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.SnapshotMock.RegionData.html @@ -70,826 +70,837 @@ 062import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; 063import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; 064import org.apache.hadoop.hbase.client.SnapshotDescription; -065import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos; -066import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest; -067import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsSnapshotDoneRequest; -068import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsSnapshotDoneResponse; -069import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; -070import org.apache.hadoop.hbase.regionserver.HRegionServer; -071import org.apache.hadoop.hbase.regionserver.Region; -072import org.apache.hadoop.hbase.util.Bytes; -073import org.apache.hadoop.hbase.util.FSTableDescriptors; -074import org.apache.hadoop.hbase.util.FSVisitor; -075import org.apache.hadoop.hbase.util.FSUtils; -076import org.apache.hadoop.hbase.util.MD5Hash; -077import org.junit.Assert; -078 -079import com.google.protobuf.ServiceException; -080 -081/** -082 * Utilities class for snapshots -083 */ -084@InterfaceAudience.Private -085public final class SnapshotTestingUtils { -086 private static final Log LOG = LogFactory.getLog(SnapshotTestingUtils.class); -087 -088 // default number of regions (and keys) given by getSplitKeys() and createTable() -089 private static byte[] KEYS = Bytes.toBytes("0123456"); -090 -091 private SnapshotTestingUtils() { -092 // private constructor for utility class -093 } -094 -095 /** -096 * Assert that we don't have any snapshots lists -097 * -098 * @throws IOException -099 * if the admin operation fails -100 */ -101 public static void assertNoSnapshots(Admin admin) throws IOException { -102 assertEquals("Have some previous snapshots", 0, admin.listSnapshots() -103 .size()); -104 } -105 -106 /** -107 * Make sure that there is only one snapshot returned from the master and its -108 * name and table match the passed in parameters. -109 */ -110 public static List<SnapshotDescription> assertExistsMatchingSnapshot( -111 Admin admin, String snapshotName, TableName tableName) -112 throws IOException { -113 // list the snapshot -114 List<SnapshotDescription> snapshots = admin.listSnapshots(); -115 -116 List<SnapshotDescription> returnedSnapshots = new ArrayList<>(); -117 for (SnapshotDescription sd : snapshots) { -118 if (snapshotName.equals(sd.getName()) && tableName.equals(sd.getTableName())) { -119 returnedSnapshots.add(sd); -120 } -121 } -122 -123 Assert.assertTrue("No matching snapshots found.", returnedSnapshots.size()>0); -124 return returnedSnapshots; -125 } -126 -127 /** -128 * Make sure that there is only one snapshot returned from the master -129 */ -130 public static void assertOneSnapshotThatMatches(Admin admin, -131 SnapshotProtos.SnapshotDescription snapshot) throws IOException { -132 assertOneSnapshotThatMatches(admin, snapshot.getName(), TableName.valueOf(snapshot.getTable())); -133 } -134 -135 /** -136 * Make sure that there is only one snapshot returned from the master and its -137 * name and table match the passed in parameters. -138 */ -139 public static List<SnapshotDescription> assertOneSnapshotThatMatches( -140 Admin admin, String snapshotName, TableName tableName) -141 throws IOException { -142 // list the snapshot -143 List<SnapshotDescription> snapshots = admin.listSnapshots(); -144 -145 assertEquals("Should only have 1 snapshot", 1, snapshots.size()); -146 assertEquals(snapshotName, snapshots.get(0).getName()); -147 assertEquals(tableName, snapshots.get(0).getTableName()); -148 -149 return snapshots; -150 } -151 -152 /** -153 * Make sure that there is only one snapshot returned from the master and its -154 * name and table match the passed in parameters. -155 */ -156 public static List<SnapshotDescription> assertOneSnapshotThatMatches( -157 Admin admin, byte[] snapshot, TableName tableName) throws IOException { -158 return assertOneSnapshotThatMatches(admin, Bytes.toString(snapshot), -159 tableName); -160 } -161 -162 public static void confirmSnapshotValid(HBaseTestingUtility testUtil, -163 SnapshotProtos.SnapshotDescription snapshotDescriptor, TableName tableName, byte[] family) -164 throws IOException { -165 MasterFileSystem mfs = testUtil.getHBaseCluster().getMaster().getMasterFileSystem(); -166 confirmSnapshotValid(snapshotDescriptor, tableName, family, mfs.getRootDir(), -167 testUtil.getAdmin(), mfs.getFileSystem()); -168 } -169 -170 /** -171 * Confirm that the snapshot contains references to all the files that should -172 * be in the snapshot. -173 */ -174 public static void confirmSnapshotValid(SnapshotProtos.SnapshotDescription snapshotDescriptor, -175 TableName tableName, byte[] testFamily, Path rootDir, Admin admin, FileSystem fs) -176 throws IOException { -177 ArrayList nonEmptyTestFamilies = new ArrayList(1); -178 nonEmptyTestFamilies.add(testFamily); -179 confirmSnapshotValid(snapshotDescriptor, tableName, -180 nonEmptyTestFamilies, null, rootDir, admin, fs); -181 } -182 -183 /** -184 * Confirm that the snapshot has no references files but only metadata. -185 */ -186 public static void confirmEmptySnapshotValid( -187 SnapshotProtos.SnapshotDescription snapshotDescriptor, TableName tableName, -188 byte[] testFamily, Path rootDir, Admin admin, FileSystem fs) -189 throws IOException { -190 ArrayList emptyTestFamilies = new ArrayList(1); -191 emptyTestFamilies.add(testFamily); -192 confirmSnapshotValid(snapshotDescriptor, tableName, -193 null, emptyTestFamilies, rootDir, admin, fs); -194 } -195 -196 /** -197 * Confirm that the snapshot contains references to all the files that should -198 * be in the snapshot. This method also perform some redundant check like -199 * the existence of the snapshotinfo or the regioninfo which are done always -200 * by the MasterSnapshotVerifier, at the end of the snapshot operation. -201 */ -202 public static void confirmSnapshotValid( -203 SnapshotProtos.SnapshotDescription snapshotDescriptor, TableName tableName, -204 List<byte[]> nonEmptyTestFamilies, List<byte[]> emptyTestFamilies, -205 Path rootDir, Admin admin, FileSystem fs) throws IOException { -206 final Configuration conf = admin.getConfiguration(); -207 -208 // check snapshot dir -209 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir( -210 snapshotDescriptor, rootDir); -211 assertTrue(fs.exists(snapshotDir)); -212 -213 SnapshotProtos.SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); -214 -215 // Extract regions and families with store files -216 final Set<byte[]> snapshotFamilies = new TreeSet<>(Bytes.BYTES_COMPARATOR); -217 -218 SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, desc); -219 Map<String, SnapshotRegionManifest> regionManifests = manifest.getRegionManifestsMap(); -220 for (SnapshotRegionManifest regionManifest: regionManifests.values()) { -221 SnapshotReferenceUtil.visitRegionStoreFiles(regionManifest, -222 new SnapshotReferenceUtil.StoreFileVisitor() { -223 @Override -224 public void storeFile(final HRegionInfo regionInfo, final String family, -225 final SnapshotRegionManifest.StoreFile storeFile) throws IOException { -226 snapshotFamilies.add(Bytes.toBytes(family)); -227 } -228 }); -229 } -230 -231 // Verify that there are store files in the specified families -232 if (nonEmptyTestFamilies != null) { -233 for (final byte[] familyName: nonEmptyTestFamilies) { -234 assertTrue(snapshotFamilies.contains(familyName)); -235 } -236 } -237 -238 // Verify that there are no store files in the specified families -239 if (emptyTestFamilies != null) { -240 for (final byte[] familyName: emptyTestFamilies) { -241 assertFalse(snapshotFamilies.contains(familyName)); -242 } -243 } -244 -245 // check the region snapshot for all the regions -246 List<HRegionInfo> regions = admin.getTableRegions(tableName); -247 // remove the non-default regions -248 RegionReplicaUtil.removeNonDefaultRegions(regions); -249 boolean hasMob = regionManifests.containsKey(MobUtils.getMobRegionInfo(tableName) -250 .getEncodedName()); -251 if (hasMob) { -252 assertEquals(regions.size(), regionManifests.size() - 1); -253 } else { -254 assertEquals(regions.size(), regionManifests.size()); -255 } -256 -257 // Verify Regions (redundant check, see MasterSnapshotVerifier) -258 for (HRegionInfo info : regions) { -259 String regionName = info.getEncodedName(); -260 assertTrue(regionManifests.containsKey(regionName)); -261 } -262 } -263 -264 /** -265 * Helper method for testing async snapshot operations. Just waits for the -266 * given snapshot to complete on the server by repeatedly checking the master. -267 * -268 * @param master: the master running the snapshot -269 * @param snapshot: the snapshot to check -270 * @param sleep: amount to sleep between checks to see if the snapshot is done -271 * @throws ServiceException if the snapshot fails -272 * @throws org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException -273 */ -274 public static void waitForSnapshotToComplete(HMaster master, -275 SnapshotProtos.SnapshotDescription snapshot, long sleep) -276 throws org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException { -277 final IsSnapshotDoneRequest request = IsSnapshotDoneRequest.newBuilder() -278 .setSnapshot(snapshot).build(); -279 IsSnapshotDoneResponse done = IsSnapshotDoneResponse.newBuilder() -280 .buildPartial(); -281 while (!done.getDone()) { -282 done = master.getMasterRpcServices().isSnapshotDone(null, request); -283 try { -284 Thread.sleep(sleep); -285 } catch (InterruptedException e) { -286 throw new org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException(e); -287 } -288 } -289 } -290 -291 /* -292 * Take snapshot with maximum of numTries attempts, ignoring CorruptedSnapshotException -293 * except for the last CorruptedSnapshotException -294 */ -295 public static void snapshot(Admin admin, final String snapshotName, final TableName tableName, -296 final SnapshotType type, final int numTries) throws IOException { -297 int tries = 0; -298 CorruptedSnapshotException lastEx = null; -299 while (tries++ < numTries) { -300 try { -301 admin.snapshot(snapshotName, tableName, type); -302 return; -303 } catch (CorruptedSnapshotException cse) { -304 LOG.warn("Got CorruptedSnapshotException", cse); -305 lastEx = cse; -306 } -307 } -308 throw lastEx; -309 } -310 -311 public static void cleanupSnapshot(Admin admin, byte[] tableName) -312 throws IOException { -313 SnapshotTestingUtils.cleanupSnapshot(admin, Bytes.toString(tableName)); -314 } -315 -316 public static void cleanupSnapshot(Admin admin, String snapshotName) -317 throws IOException { -318 // delete the taken snapshot -319 admin.deleteSnapshot(snapshotName); -320 assertNoSnapshots(admin); -321 } -322 -323 /** -324 * Expect the snapshot to throw an error when checking if the snapshot is -325 * complete -326 * -327 * @param master master to check -328 * @param snapshot the {@link SnapshotDescription} request to pass to the master -329 * @param clazz expected exception from the master -330 */ -331 public static void expectSnapshotDoneException(HMaster master, -332 IsSnapshotDoneRequest snapshot, -333 Class<? extends HBaseSnapshotException> clazz) { -334 try { -335 master.getMasterRpcServices().isSnapshotDone(null, snapshot); -336 Assert.fail("didn't fail to lookup a snapshot"); -337 } catch (org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException se) { -338 try { -339 throw ProtobufUtil.handleRemoteException(se); -340 } catch (HBaseSnapshotException e) { -341 assertEquals("Threw wrong snapshot exception!", clazz, e.getClass()); -342 } catch (Throwable t) { -343 Assert.fail("Threw an unexpected exception:" + t); -344 } -345 } -346 } -347 -348 /** -349 * List all the HFiles in the given table -350 * -351 * @param fs: FileSystem where the table lives -352 * @param tableDir directory of the table -353 * @return array of the current HFiles in the table (could be a zero-length array) -354 * @throws IOException on unexecpted error reading the FS -355 */ -356 public static ArrayList<String> listHFileNames(final FileSystem fs, final Path tableDir) -357 throws IOException { -358 final ArrayList<String> hfiles = new ArrayList<>(); -359 FSVisitor.visitTableStoreFiles(fs, tableDir, new FSVisitor.StoreFileVisitor() { -360 @Override -361 public void storeFile(final String region, final String family, final String hfileName) -362 throws IOException { -363 hfiles.add(hfileName); -364 } -365 }); -366 Collections.sort(hfiles); -367 return hfiles; -368 } -369 -370 /** -371 * Take a snapshot of the specified table and verify that the given family is -372 * not empty. Note that this will leave the table disabled -373 * in the case of an offline snapshot. -374 */ -375 public static void createSnapshotAndValidate(Admin admin, -376 TableName tableName, String familyName, String snapshotNameString, -377 Path rootDir, FileSystem fs, boolean onlineSnapshot) -378 throws Exception { -379 ArrayList<byte[]> nonEmptyFamilyNames = new ArrayList<>(1); -380 nonEmptyFamilyNames.add(Bytes.toBytes(familyName)); -381 createSnapshotAndValidate(admin, tableName, nonEmptyFamilyNames, /* emptyFamilyNames= */ null, -382 snapshotNameString, rootDir, fs, onlineSnapshot); -383 } -384 -385 /** -386 * Take a snapshot of the specified table and verify the given families. -387 * Note that this will leave the table disabled in the case of an offline snapshot. -388 */ -389 public static void createSnapshotAndValidate(Admin admin, -390 TableName tableName, List<byte[]> nonEmptyFamilyNames, List<byte[]> emptyFamilyNames, -391 String snapshotNameString, Path rootDir, FileSystem fs, boolean onlineSnapshot) -392 throws Exception { -393 if (!onlineSnapshot) { -394 try { -395 admin.disableTable(tableName); -396 } catch (TableNotEnabledException tne) { -397 LOG.info("In attempting to disable " + tableName + " it turns out that the this table is " + -398 "already disabled."); -399 } -400 } -401 admin.snapshot(snapshotNameString, tableName); -402 -403 List<SnapshotDescription> snapshots = -404 SnapshotTestingUtils.assertExistsMatchingSnapshot(admin, snapshotNameString, tableName); -405 if (snapshots == null || snapshots.size() != 1) { -406 Assert.fail("Incorrect number of snapshots for table " + tableName); -407 } -408 -409 SnapshotTestingUtils.confirmSnapshotValid( -410 ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), tableName, nonEmptyFamilyNames, -411 emptyFamilyNames, rootDir, admin, fs); -412 } +065import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo; +066import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos; +067import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest; +068import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsSnapshotDoneRequest; +069import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsSnapshotDoneResponse; +070import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; +071import org.apache.hadoop.hbase.regionserver.HRegionServer; +072import org.apache.hadoop.hbase.regionserver.Region; +073import org.apache.hadoop.hbase.util.Bytes; +074import org.apache.hadoop.hbase.util.FSTableDescriptors; +075import org.apache.hadoop.hbase.util.FSVisitor; +076import org.apache.hadoop.hbase.util.FSUtils; +077import org.apache.hadoop.hbase.util.MD5Hash; +078import org.junit.Assert; +079 +080import com.google.protobuf.ServiceException; +081 +082/** +083 * Utilities class for snapshots +084 */ +085@InterfaceAudience.Private +086public final class SnapshotTestingUtils { +087 private static final Log LOG = LogFactory.getLog(SnapshotTestingUtils.class); +088 +089 // default number of regions (and keys) given by getSplitKeys() and createTable() +090 private static byte[] KEYS = Bytes.toBytes("0123456"); +091 +092 private SnapshotTestingUtils() { +093 // private constructor for utility class +094 } +095 +096 /** +097 * Assert that we don't have any snapshots lists +098 * +099 * @throws IOException +100 * if the admin operation fails +101 */ +102 public static void assertNoSnapshots(Admin admin) throws IOException { +103 assertEquals("Have some previous snapshots", 0, admin.listSnapshots() +104 .size()); +105 } +106 +107 /** +108 * Make sure that there is only one snapshot returned from the master and its +109 * name and table match the passed in parameters. +110 */ +111 public static List<SnapshotDescription> assertExistsMatchingSnapshot( +112 Admin admin, String snapshotName, TableName tableName) +113 throws IOException { +114 // list the snapshot +115 List<SnapshotDescription> snapshots = admin.listSnapshots(); +116 +117 List<SnapshotDescription> returnedSnapshots = new ArrayList<>(); +118 for (SnapshotDescription sd : snapshots) { +119 if (snapshotName.equals(sd.getName()) && tableName.equals(sd.getTableName())) { +120 returnedSnapshots.add(sd); +121 } +122 } +123 +124 Assert.assertTrue("No matching snapshots found.", returnedSnapshots.size()>0); +125 return returnedSnapshots; +126 } +127 +128 /** +129 * Make sure that there is only one snapshot returned from the master +130 */ +131 public static void assertOneSnapshotThatMatches(Admin admin, +132 SnapshotProtos.SnapshotDescription snapshot) throws IOException { +133 assertOneSnapshotThatMatches(admin, snapshot.getName(), TableName.valueOf(snapshot.getTable())); +134 } +135 +136 /** +137 * Make sure that there is only one snapshot returned from the master and its +138 * name and table match the passed in parameters. +139 */ +140 public static List<SnapshotDescription> assertOneSnapshotThatMatches( +141 Admin admin, String snapshotName, TableName tableName) +142 throws IOException { +143 // list the snapshot +144 List<SnapshotDescription> snapshots = admin.listSnapshots(); +145 +146 assertEquals("Should only have 1 snapshot", 1, snapshots.size()); +147 assertEquals(snapshotName, snapshots.get(0).getName()); +148 assertEquals(tableName, snapshots.get(0).getTableName()); +149 +150 return snapshots; +151 } +152 +153 /** +154 * Make sure that there is only one snapshot returned from the master and its +155 * name and table match the passed in parameters. +156 */ +157 public static List<SnapshotDescription> assertOneSnapshotThatMatches( +158 Admin admin, byte[] snapshot, TableName tableName) throws IOException { +159 return assertOneSnapshotThatMatches(admin, Bytes.toString(snapshot), +160 tableName); +161 } +162 +163 public static void confirmSnapshotValid(HBaseTestingUtility testUtil, +164 SnapshotProtos.SnapshotDescription snapshotDescriptor, TableName tableName, byte[] family) +165 throws IOException { +166 MasterFileSystem mfs = testUtil.getHBaseCluster().getMaster().getMasterFileSystem(); +167 confirmSnapshotValid(snapshotDescriptor, tableName, family, mfs.getRootDir(), +168 testUtil.getAdmin(), mfs.getFileSystem()); +169 } +170 +171 /** +172 * Confirm that the snapshot contains references to all the files that should +173 * be in the snapshot. +174 */ +175 public static void confirmSnapshotValid(SnapshotProtos.SnapshotDescription snapshotDescriptor, +176 TableName tableName, byte[] testFamily, Path rootDir, Admin admin, FileSystem fs) +177 throws IOException { +178 ArrayList nonEmptyTestFamilies = new ArrayList(1); +179 nonEmptyTestFamilies.add(testFamily); +180 confirmSnapshotValid(snapshotDescriptor, tableName, +181 nonEmptyTestFamilies, null, rootDir, admin, fs); +182 } +183 +184 /** +185 * Confirm that the snapshot has no references files but only metadata. +186 */ +187 public static void confirmEmptySnapshotValid( +188 SnapshotProtos.SnapshotDescription snapshotDescriptor, TableName tableName, +189 byte[] testFamily, Path rootDir, Admin admin, FileSystem fs) +190 throws IOException { +191 ArrayList emptyTestFamilies = new ArrayList(1); +192 emptyTestFamilies.add(testFamily); +193 confirmSnapshotValid(snapshotDescriptor, tableName, +194 null, emptyTestFamilies, rootDir, admin, fs); +195 } +196 +197 /** +198 * Confirm that the snapshot contains references to all the files that should +199 * be in the snapshot. This method also perform some redundant check like +200 * the existence of the snapshotinfo or the regioninfo which are done always +201 * by the MasterSnapshotVerifier, at the end of the snapshot operation. +202 */ +203 public static void confirmSnapshotValid( +204 SnapshotProtos.SnapshotDescription snapshotDescriptor, TableName tableName, +205 List<byte[]> nonEmptyTestFamilies, List<byte[]> emptyTestFamilies, +206 Path rootDir, Admin admin, FileSystem fs) throws IOException { +207 final Configuration conf = admin.getConfiguration(); +208 +209 // check snapshot dir +210 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir( +211 snapshotDescriptor, rootDir); +212 assertTrue(fs.exists(snapshotDir)); +213 +214 SnapshotProtos.SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); +215 +216 // Extract regions and families with store files +217 final Set<byte[]> snapshotFamilies = new TreeSet<>(Bytes.BYTES_COMPARATOR); +218 +219 SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, desc); +220 Map<String, SnapshotRegionManifest> regionManifests = manifest.getRegionManifestsMap(); +221 for (SnapshotRegionManifest regionManifest: regionManifests.values()) { +222 SnapshotReferenceUtil.visitRegionStoreFiles(regionManifest, +223 new SnapshotReferenceUtil.StoreFileVisitor() { +224 @Override +225 public void storeFile(final HRegionInfo regionInfo, final String family, +226 final SnapshotRegionManifest.StoreFile storeFile) throws IOException { +227 snapshotFamilies.add(Bytes.toBytes(family)); +228 } +229 }); +230 } +231 +232 // Verify that there are store files in the specified families +233 if (nonEmptyTestFamilies != null) { +234 for (final byte[] familyName: nonEmptyTestFamilies) { +235 assertTrue(snapshotFamilies.contains(familyName)); +236 } +237 } +238 +239 // Verify that there are no store files in the specified families +240 if (emptyTestFamilies != null) { +241 for (final byte[] familyName: emptyTestFamilies) { +242 assertFalse(snapshotFamilies.contains(familyName)); +243 } +244 } +245 +246 // check the region snapshot for all the regions +247 List<HRegionInfo> regions = admin.getTableRegions(tableName); +248 // remove the non-default regions +249 RegionReplicaUtil.removeNonDefaultRegions(regions); +250 boolean hasMob = regionManifests.containsKey(MobUtils.getMobRegionInfo(tableName) +251 .getEncodedName()); +252 if (hasMob) { +253 assertEquals(regions.size(), regionManifests.size() - 1); +254 } else { +255 // if create snapshot when table splitting, parent region will be included to the snapshot +256 // region manifest. we should exclude the parent regions. +257 int regionCountExclusiveSplitParent = 0; +258 for (SnapshotRegionManifest snapshotRegionManifest : regionManifests.values()) { +259 HRegionInfo hri = HRegionInfo.convert(snapshotRegionManifest.getRegionInfo()); +260 if (hri.isOffline() && (hri.isSplit() || hri.isSplitParent())) { +261 continue; +262 } +263 regionCountExclusiveSplitParent++; +264 } +265 assertEquals(regions.size(), regionCountExclusiveSplitParent); +266 } +267 +268 // Verify Regions (redundant check, see MasterSnapshotVerifier) +269 for (HRegionInfo info : regions) { +270 String regionName = info.getEncodedName(); +271 assertTrue(regionManifests.containsKey(regionName)); +272 } +273 } +274 +275 /** +276 * Helper method for testing async snapshot operations. Just waits for the +277 * given snapshot to complete on the server by repeatedly checking the master. +278 * +279 * @param master: the master running the snapshot +280 * @param snapshot: the snapshot to check +281 * @param sleep: amount to sleep between checks to see if the snapshot is done +282 * @throws ServiceException if the snapshot fails +283 * @throws org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException +284 */ +285 public static void waitForSnapshotToComplete(HMaster master, +286 SnapshotProtos.SnapshotDescription snapshot, long sleep) +287 throws org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException { +288 final IsSnapshotDoneRequest request = IsSnapshotDoneRequest.newBuilder() +289 .setSnapshot(snapshot).build(); +290 IsSnapshotDoneResponse done = IsSnapshotDoneResponse.newBuilder() +291 .buildPartial(); +292 while (!done.getDone()) { +293 done = master.getMasterRpcServices().isSnapshotDone(null, request); +294 try { +295 Thread.sleep(sleep); +296 } catch (InterruptedException e) { +297 throw new org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException(e); +298 } +299 } +300 } +301 +302 /* +303 * Take snapshot with maximum of numTries attempts, ignoring CorruptedSnapshotException +304 * except for the last CorruptedSnapshotException +305 */ +306 public static void snapshot(Admin admin, final String snapshotName, final TableName tableName, +307 final SnapshotType type, final int numTries) throws IOException { +308 int tries = 0; +309 CorruptedSnapshotException lastEx = null; +310 while (tries++ < numTries) { +311 try { +312 admin.snapshot(snapshotName, tableName, type); +313 return; +314 } catch (CorruptedSnapshotException cse) { +315 LOG.warn("Got CorruptedSnapshotException", cse); +316 lastEx = cse; +317 } +318 } +319 throw lastEx; +320 } +321 +322 public static void cleanupSnapshot(Admin admin, byte[] tableName) +323 throws IOException { +324 SnapshotTestingUtils.cleanupSnapshot(admin, Bytes.toString(tableName)); +325 } +326 +327 public static void cleanupSnapshot(Admin admin, String snapshotName) +328 throws IOException { +329 // delete the taken snapshot +330 admin.deleteSnapshot(snapshotName); +331 assertNoSnapshots(admin); +332 } +333 +334 /** +335 * Expect the snapshot to throw an error when checking if the snapshot is +336 * complete +337 * +338 * @param master master to check +339 * @param snapshot the {@link SnapshotDescription} request to pass to the master +340 * @param clazz expected exception from the master +341 */ +342 public static void expectSnapshotDoneException(HMaster master, +343 IsSnapshotDoneRequest snapshot, +344 Class<? extends HBaseSnapshotException> clazz) { +345 try { +346 master.getMasterRpcServices().isSnapshotDone(null, snapshot); +347 Assert.fail("didn't fail to lookup a snapshot"); +348 } catch (org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException se) { +349 try { +350 throw ProtobufUtil.handleRemoteException(se); +351 } catch (HBaseSnapshotException e) { +352 assertEquals("Threw wrong snapshot exception!", clazz, e.getClass()); +353 } catch (Throwable t) { +354 Assert.fail("Threw an unexpected exception:" + t); +355 } +356 } +357 } +358 +359 /** +360 * List all the HFiles in the given table +361 * +362 * @param fs: FileSystem where the table lives +363 * @param tableDir directory of the table +364 * @return array of the current HFiles in the table (could be a zero-length array) +365 * @throws IOException on unexecpted error reading the FS +366 */ +367 public static ArrayList<String> listHFileNames(final FileSystem fs, final Path tableDir) +368 throws IOException { +369 final ArrayList<String> hfiles = new ArrayList<>(); +370 FSVisitor.visitTableStoreFiles(fs, tableDir, new FSVisitor.StoreFileVisitor() { +371 @Override +372 public void storeFile(final String region, final String family, final String hfileName) +373 throws IOException { +374 hfiles.add(hfileName); +375 } +376 }); +377 Collections.sort(hfiles); +378 return hfiles; +379 } +380 +381 /** +382 * Take a snapshot of the specified table and verify that the given family is +383 * not empty. Note that this will leave the table disabled +384 * in the case of an offline snapshot. +385 */ +386 public static void createSnapshotAndValidate(Admin admin, +387 TableName tableName, String familyName, String snapshotNameString, +388 Path rootDir, FileSystem fs, boolean onlineSnapshot) +389 throws Exception { +390 ArrayList<byte[]> nonEmptyFamilyNames = new ArrayList<>(1); +391 nonEmptyFamilyNames.add(Bytes.toBytes(familyName)); +392 createSnapshotAndValidate(admin, tableName, nonEmptyFamilyNames, /* emptyFamilyNames= */ null, +393 snapshotNameString, rootDir, fs, onlineSnapshot); +394 } +395 +396 /** +397 * Take a snapshot of the specified table and verify the given families. +398 * Note that this will leave the table disabled in the case of an offline snapshot. +399 */ +400 public static void createSnapshotAndValidate(Admin admin, +401 TableName tableName, List<byte[]> nonEmptyFamilyNames, List<byte[]> emptyFamilyNames, +402 String snapshotNameString, Path rootDir, FileSystem fs, boolean onlineSnapshot) +403 throws Exception { +404 if (!onlineSnapshot) { +405 try { +406 admin.disableTable(tableName); +407 } catch (TableNotEnabledException tne) { +408 LOG.info("In attempting to disable " + tableName + " it turns out that the this table is " + +409 "already disabled."); +410 } +411 } +412 admin.snapshot(snapshotNameString, tableName); 413 -414 /** -415 * Corrupt the specified snapshot by deleting some files. -416 * -417 * @param util {@link HBaseTestingUtility} -418 * @param snapshotName name of the snapshot to corrupt -419 * @return array of the corrupted HFiles -420 * @throws IOException on unexecpted error reading the FS -421 */ -422 public static ArrayList corruptSnapshot(final HBaseTestingUtility util, final String snapshotName) -423 throws IOException { -424 final MasterFileSystem mfs = util.getHBaseCluster().getMaster().getMasterFileSystem(); -425 final FileSystem fs = mfs.getFileSystem(); -426 -427 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, -428 mfs.getRootDir()); -429 SnapshotProtos.SnapshotDescription snapshotDesc = -430 SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); -431 final TableName table = TableName.valueOf(snapshotDesc.getTable()); -432 -433 final ArrayList corruptedFiles = new ArrayList(); -434 final Configuration conf = util.getConfiguration(); -435 SnapshotReferenceUtil.visitTableStoreFiles(conf, fs, snapshotDir, snapshotDesc, -436 new SnapshotReferenceUtil.StoreFileVisitor() { -437 @Override -438 public void storeFile(final HRegionInfo regionInfo, final String family, -439 final SnapshotRegionManifest.StoreFile storeFile) throws IOException { -440 String region = regionInfo.getEncodedName(); -441 String hfile = storeFile.getName(); -442 HFileLink link = HFileLink.build(conf, table, region, family, hfile); -443 if (corruptedFiles.size() % 2 == 0) { -444 fs.delete(link.getAvailablePath(fs), true); -445 corruptedFiles.add(hfile); -446 } -447 } -448 }); -449 -450 assertTrue(corruptedFiles.size() > 0); -451 return corruptedFiles; -452 } -453 -454 // ========================================================================== -455 // Snapshot Mock -456 // ========================================================================== -457 public static class SnapshotMock { -458 protected final static String TEST_FAMILY = "cf"; -459 public final static int TEST_NUM_REGIONS = 4; +414 List<SnapshotDescription> snapshots = +415 SnapshotTestingUtils.assertExistsMatchingSnapshot(admin, snapshotNameString, tableName); +416 if (snapshots == null || snapshots.size() != 1) { +417 Assert.fail("Incorrect number of snapshots for table " + tableName); +418 } +419 +420 SnapshotTestingUtils.confirmSnapshotValid( +421 ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), tableName, nonEmptyFamilyNames, +422 emptyFamilyNames, rootDir, admin, fs); +423 } +424 +425 /** +426 * Corrupt the specified snapshot by deleting some files. +427 * +428 * @param util {@link HBaseTestingUtility} +429 * @param snapshotName name of the snapshot to corrupt +430 * @return array of the corrupted HFiles +431 * @throws IOException on unexecpted error reading the FS +432 */ +433 public static ArrayList corruptSnapshot(final HBaseTestingUtility util, final String snapshotName) +434 throws IOException { +435 final MasterFileSystem mfs = util.getHBaseCluster().getMaster().getMasterFileSystem(); +436 final FileSystem fs = mfs.getFileSystem(); +437 +438 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, +439 mfs.getRootDir()); +440 SnapshotProtos.SnapshotDescription snapshotDesc = +441 SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); +442 final TableName table = TableName.valueOf(snapshotDesc.getTable()); +443 +444 final ArrayList corruptedFiles = new ArrayList(); +445 final Configuration conf = util.getConfiguration(); +446 SnapshotReferenceUtil.visitTableStoreFiles(conf, fs, snapshotDir, snapshotDesc, +447 new SnapshotReferenceUtil.StoreFileVisitor() { +448 @Override +449 public void storeFile(final HRegionInfo regionInfo, final String family, +450 final SnapshotRegionManifest.StoreFile storeFile) throws IOException { +451 String region = regionInfo.getEncodedName(); +452 String hfile = storeFile.getName(); +453 HFileLink link = HFileLink.build(conf, table, region, family, hfile); +454 if (corruptedFiles.size() % 2 == 0) { +455 fs.delete(link.getAvailablePath(fs), true); +456 corruptedFiles.add(hfile); +457 } +458 } +459 }); 460 -461 private final Configuration conf; -462 private final FileSystem fs; -463 private final Path rootDir; +461 assertTrue(corruptedFiles.size() > 0); +462 return corruptedFiles; +463 } 464 -465 static class RegionData { -466 public HRegionInfo hri; -467 public Path tableDir; -468 public Path[] files; -469 -470 public RegionData(final Path tableDir, final HRegionInfo hri, final int nfiles) { -471 this.tableDir = tableDir; -472 this.hri = hri; -473 this.files = new Path[nfiles]; -474 } -475 } -476 -477 public static class SnapshotBuilder { -478 private final RegionData[] tableRegions; -479 private final SnapshotProtos.SnapshotDescription desc; -480 private final HTableDescriptor htd; -481 private final Configuration conf; -482 private final FileSystem fs; -483 private final Path rootDir; -484 private Path snapshotDir; -485 private int snapshotted = 0; -486 -487 public SnapshotBuilder(final Configuration conf, final FileSystem fs, -488 final Path rootDir, final HTableDescriptor htd, -489 final SnapshotProtos.SnapshotDescription desc, final RegionData[] tableRegions) -490 throws IOException { -491 this.fs = fs; -492 this.conf = conf; -493 this.rootDir = rootDir; -494 this.htd = htd; -495 this.desc = desc; -496 this.tableRegions = tableRegions; -497 this.snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir); -498 new FSTableDescriptors(conf) -499 .createTableDescriptorForTableDirectory(snapshotDir, htd, false); -500 } -501 -502 public HTableDescriptor getTableDescriptor() { -503 return this.htd; -504 } -505 -506 public SnapshotProtos.SnapshotDescription getSnapshotDescription() { -507 return this.desc; -508 } -509 -510 public Path getSnapshotsDir() { -511 return this.snapshotDir; -512 } -513 -514 public Path[] addRegion() throws IOException { -515 return addRegion(desc); -516 } -517 -518 public Path[] addRegionV1() throws IOException { -519 return addRegion(desc.toBuilder() -520 .setVersion(SnapshotManifestV1.DESCRIPTOR_VERSION) -521 .build()); -522 } -523 -524 public Path[] addRegionV2() throws IOException { -525 return addRegion(desc.toBuilder() -526 .setVersion(SnapshotManifestV2.DESCRIPTOR_VERSION) -527 .build()); -528 } -529 -530 private Path[] addRegion(final SnapshotProtos.SnapshotDescription desc) throws IOException { -531 if (this.snapshotted == tableRegions.length) { -532 throw new UnsupportedOperationException("No more regions in the table"); -533 } +465 // ========================================================================== +466 // Snapshot Mock +467 // ========================================================================== +