Return-Path: X-Original-To: apmail-hbase-commits-archive@www.apache.org Delivered-To: apmail-hbase-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 1380E115DC for ; Sun, 11 May 2014 04:30:00 +0000 (UTC) Received: (qmail 60883 invoked by uid 500); 10 May 2014 23:30:36 -0000 Delivered-To: apmail-hbase-commits-archive@hbase.apache.org Received: (qmail 57463 invoked by uid 500); 10 May 2014 23:23:49 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 76294 invoked by uid 99); 10 May 2014 23:09:31 -0000 Received: from Unknown (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 10 May 2014 23:09:31 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 07 May 2014 21:28:41 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id BD50A2388B1B; Wed, 7 May 2014 21:28:15 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1593139 [5/6] - in /hbase/trunk: hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ hbase-protocol/src/main/protobuf/ hbase-server/src/main/java/org/apache/hadoop/hbase/client/ hbase-server/src/main/java/org/apache/ha... Date: Wed, 07 May 2014 21:28:13 -0000 To: commits@hbase.apache.org From: mbertozzi@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20140507212815.BD50A2388B1B@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java?rev=1593139&r1=1593138&r2=1593139&view=diff ============================================================================== --- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java (original) +++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java Wed May 7 21:28:12 2014 @@ -19,24 +19,28 @@ package org.apache.hadoop.hbase.snapshot; import java.io.IOException; -import java.io.FileNotFoundException; +import java.io.InterruptedIOException; import java.util.HashSet; -import java.util.LinkedList; import java.util.List; -import java.util.Map; import java.util.Set; -import java.util.TreeMap; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorCompletionService; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.io.HFileLink; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest; import org.apache.hadoop.hbase.regionserver.wal.HLogUtil; -import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.hbase.util.FSVisitor; /** @@ -44,8 +48,15 @@ import org.apache.hadoop.hbase.util.FSVi */ @InterfaceAudience.Private public final class SnapshotReferenceUtil { - public interface FileVisitor extends FSVisitor.StoreFileVisitor, - FSVisitor.RecoveredEditsVisitor, FSVisitor.LogFileVisitor { + public static final Log LOG = LogFactory.getLog(SnapshotReferenceUtil.class); + + public interface StoreFileVisitor { + void storeFile(final HRegionInfo regionInfo, final String familyName, + final SnapshotRegionManifest.StoreFile storeFile) throws IOException; + } + + public interface SnapshotVisitor extends StoreFileVisitor, + FSVisitor.LogFileVisitor { } private SnapshotReferenceUtil() { @@ -64,80 +75,79 @@ public final class SnapshotReferenceUtil } /** - * Get the snapshotted recovered.edits dir for the specified region. - * - * @param snapshotDir directory where the specific snapshot is stored - * @param regionName name of the region - * @return path to the recovered.edits directory for the specified region files. - */ - public static Path getRecoveredEditsDir(Path snapshotDir, String regionName) { - return HLogUtil.getRegionDirRecoveredEditsDir(new Path(snapshotDir, regionName)); - } - - /** - * Get the snapshot recovered.edits file + * Iterate over the snapshot store files, restored.edits and logs * - * @param snapshotDir directory where the specific snapshot is stored - * @param regionName name of the region - * @param logfile name of the edit file - * @return full path of the log file for the specified region files. + * @param conf The current {@link Configuration} instance. + * @param fs {@link FileSystem} + * @param snapshotDir {@link Path} to the Snapshot directory + * @param visitor callback object to get the referenced files + * @throws IOException if an error occurred while scanning the directory */ - public static Path getRecoveredEdits(Path snapshotDir, String regionName, String logfile) { - return new Path(getRecoveredEditsDir(snapshotDir, regionName), logfile); + public static void visitReferencedFiles(final Configuration conf, final FileSystem fs, + final Path snapshotDir, final SnapshotVisitor visitor) + throws IOException { + SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); + visitReferencedFiles(conf, fs, snapshotDir, desc, visitor); } /** * Iterate over the snapshot store files, restored.edits and logs * + * @param conf The current {@link Configuration} instance. * @param fs {@link FileSystem} * @param snapshotDir {@link Path} to the Snapshot directory + * @param desc the {@link SnapshotDescription} of the snapshot to verify * @param visitor callback object to get the referenced files * @throws IOException if an error occurred while scanning the directory */ - public static void visitReferencedFiles(final FileSystem fs, final Path snapshotDir, - final FileVisitor visitor) throws IOException { - visitTableStoreFiles(fs, snapshotDir, visitor); - visitRecoveredEdits(fs, snapshotDir, visitor); + public static void visitReferencedFiles(final Configuration conf, final FileSystem fs, + final Path snapshotDir, final SnapshotDescription desc, final SnapshotVisitor visitor) + throws IOException { + visitTableStoreFiles(conf, fs, snapshotDir, desc, visitor); visitLogFiles(fs, snapshotDir, visitor); } /** * Iterate over the snapshot store files * + * @param conf The current {@link Configuration} instance. * @param fs {@link FileSystem} * @param snapshotDir {@link Path} to the Snapshot directory + * @param desc the {@link SnapshotDescription} of the snapshot to verify * @param visitor callback object to get the store files * @throws IOException if an error occurred while scanning the directory */ - public static void visitTableStoreFiles(final FileSystem fs, final Path snapshotDir, - final FSVisitor.StoreFileVisitor visitor) throws IOException { - FSVisitor.visitTableStoreFiles(fs, snapshotDir, visitor); + public static void visitTableStoreFiles(final Configuration conf, final FileSystem fs, + final Path snapshotDir, final SnapshotDescription desc, final StoreFileVisitor visitor) + throws IOException { + SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, desc); + List regionManifests = manifest.getRegionManifests(); + if (regionManifests == null || regionManifests.size() == 0) { + LOG.debug("No manifest files present: " + snapshotDir); + return; + } + + for (SnapshotRegionManifest regionManifest: regionManifests) { + visitRegionStoreFiles(regionManifest, visitor); + } } /** * Iterate over the snapshot store files in the specified region * - * @param fs {@link FileSystem} - * @param regionDir {@link Path} to the Snapshot region directory + * @param manifest snapshot manifest to inspect * @param visitor callback object to get the store files * @throws IOException if an error occurred while scanning the directory */ - public static void visitRegionStoreFiles(final FileSystem fs, final Path regionDir, - final FSVisitor.StoreFileVisitor visitor) throws IOException { - FSVisitor.visitRegionStoreFiles(fs, regionDir, visitor); - } - - /** - * Iterate over the snapshot recovered.edits - * - * @param fs {@link FileSystem} - * @param snapshotDir {@link Path} to the Snapshot directory - * @param visitor callback object to get the recovered.edits files - * @throws IOException if an error occurred while scanning the directory - */ - public static void visitRecoveredEdits(final FileSystem fs, final Path snapshotDir, - final FSVisitor.RecoveredEditsVisitor visitor) throws IOException { - FSVisitor.visitTableRecoveredEdits(fs, snapshotDir, visitor); + public static void visitRegionStoreFiles(final SnapshotRegionManifest manifest, + final StoreFileVisitor visitor) throws IOException { + HRegionInfo regionInfo = HRegionInfo.convert(manifest.getRegionInfo()); + for (SnapshotRegionManifest.FamilyFiles familyFiles: manifest.getFamilyFilesList()) { + String familyName = familyFiles.getFamilyName().toStringUtf8(); + for (SnapshotRegionManifest.StoreFile storeFile: familyFiles.getStoreFilesList()) { + visitor.storeFile(regionInfo, familyName, storeFile); + } + } } /** @@ -165,85 +175,151 @@ public final class SnapshotReferenceUtil */ public static void verifySnapshot(final Configuration conf, final FileSystem fs, final Path snapshotDir, final SnapshotDescription snapshotDesc) throws IOException { - final TableName table = TableName.valueOf(snapshotDesc.getTable()); - visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() { - public void storeFile (final String region, final String family, final String hfile) - throws IOException { - HFileLink link = HFileLink.create(conf, table, region, family, hfile); - try { - link.getFileStatus(fs); - } catch (FileNotFoundException e) { - throw new CorruptedSnapshotException("Corrupted snapshot '" + snapshotDesc + "'", e); + SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc); + verifySnapshot(conf, fs, manifest); + } + + /** + * Verify the validity of the snapshot + * + * @param conf The current {@link Configuration} instance. + * @param fs {@link FileSystem} + * @param manifest snapshot manifest to inspect + * @throws CorruptedSnapshotException if the snapshot is corrupted + * @throws IOException if an error occurred while scanning the directory + */ + public static void verifySnapshot(final Configuration conf, final FileSystem fs, + final SnapshotManifest manifest) throws IOException { + final SnapshotDescription snapshotDesc = manifest.getSnapshotDescription(); + final Path snapshotDir = manifest.getSnapshotDir(); + + List regionManifests = manifest.getRegionManifests(); + if (regionManifests == null || regionManifests.size() == 0) { + LOG.debug("No manifest files present: " + snapshotDir); + return; + } + + ExecutorService exec = SnapshotManifest.createExecutor(conf, "VerifySnapshot"); + final ExecutorCompletionService completionService = + new ExecutorCompletionService(exec); + try { + for (final SnapshotRegionManifest regionManifest: regionManifests) { + completionService.submit(new Callable() { + @Override + public Void call() throws IOException { + visitRegionStoreFiles(regionManifest, new StoreFileVisitor() { + @Override + public void storeFile(final HRegionInfo regionInfo, final String family, + final SnapshotRegionManifest.StoreFile storeFile) throws IOException { + verifyStoreFile(conf, fs, snapshotDir, snapshotDesc, regionInfo, family, storeFile); + } + }); + return null; + } + }); + } + try { + for (int i = 0; i < regionManifests.size(); ++i) { + completionService.take().get(); + } + } catch (InterruptedException e) { + throw new InterruptedIOException(e.getMessage()); + } catch (ExecutionException e) { + if (e.getCause() instanceof CorruptedSnapshotException) { + throw new CorruptedSnapshotException(e.getCause().getMessage(), snapshotDesc); + } else { + IOException ex = new IOException(); + ex.initCause(e.getCause()); + throw ex; } } - }); + } finally { + exec.shutdown(); + } } /** - * Returns the set of region names available in the snapshot. + * Verify the validity of the snapshot store file * + * @param conf The current {@link Configuration} instance. * @param fs {@link FileSystem} - * @param snapshotDir {@link Path} to the Snapshot directory + * @param snapshotDir {@link Path} to the Snapshot directory of the snapshot to verify + * @param snapshot the {@link SnapshotDescription} of the snapshot to verify + * @param regionInfo {@link HRegionInfo} of the region that contains the store file + * @param family family that contains the store file + * @param storeFile the store file to verify + * @throws CorruptedSnapshotException if the snapshot is corrupted * @throws IOException if an error occurred while scanning the directory - * @return the set of the regions contained in the snapshot */ - public static Set getSnapshotRegionNames(final FileSystem fs, final Path snapshotDir) - throws IOException { - FileStatus[] regionDirs = FSUtils.listStatus(fs, snapshotDir, new FSUtils.RegionDirFilter(fs)); - if (regionDirs == null) return null; + public static void verifyStoreFile(final Configuration conf, final FileSystem fs, + final Path snapshotDir, final SnapshotDescription snapshot, final HRegionInfo regionInfo, + final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException { + String fileName = storeFile.getName(); + + Path refPath = null; + if (StoreFileInfo.isReference(fileName)) { + // If is a reference file check if the parent file is present in the snapshot + Path snapshotHFilePath = new Path(new Path( + new Path(snapshotDir, regionInfo.getEncodedName()), family), fileName); + refPath = StoreFileInfo.getReferredToFile(snapshotHFilePath); + if (!fs.exists(refPath)) { + throw new CorruptedSnapshotException("Missing parent hfile for: " + fileName, snapshot); + } + } + + Path linkPath; + if (refPath != null && HFileLink.isHFileLink(refPath)) { + linkPath = new Path(family, refPath.getName()); + } else if (HFileLink.isHFileLink(fileName)) { + linkPath = new Path(family, fileName); + } else { + linkPath = new Path(family, HFileLink.createHFileLinkName( + TableName.valueOf(snapshot.getTable()), regionInfo.getEncodedName(), fileName)); + } - Set regions = new HashSet(); - for (FileStatus regionDir: regionDirs) { - regions.add(regionDir.getPath().getName()); + // check if the linked file exists (in the archive, or in the table dir) + HFileLink link = new HFileLink(conf, linkPath); + if (!link.exists(fs)) { + throw new CorruptedSnapshotException("Can't find hfile: " + fileName + + " in the real (" + link.getOriginPath() + ") or archive (" + link.getArchivePath() + + ") directory for the primary table.", snapshot); } - return regions; } /** - * Get the list of hfiles for the specified snapshot region. - * NOTE: The current implementation keeps one empty file per HFile in the region. - * The file name matches the one in the original table, and by reconstructing - * the path you can quickly jump to the referenced file. + * Returns the store file names in the snapshot. * + * @param conf The current {@link Configuration} instance. * @param fs {@link FileSystem} - * @param snapshotRegionDir {@link Path} to the Snapshot region directory - * @return Map of hfiles per family, the key is the family name and values are hfile names + * @param snapshotDir {@link Path} to the Snapshot directory * @throws IOException if an error occurred while scanning the directory + * @return the names of hfiles in the specified snaphot */ - public static Map> getRegionHFileReferences(final FileSystem fs, - final Path snapshotRegionDir) throws IOException { - final Map> familyFiles = new TreeMap>(); - - visitRegionStoreFiles(fs, snapshotRegionDir, - new FSVisitor.StoreFileVisitor() { - public void storeFile (final String region, final String family, final String hfile) - throws IOException { - List hfiles = familyFiles.get(family); - if (hfiles == null) { - hfiles = new LinkedList(); - familyFiles.put(family, hfiles); - } - hfiles.add(hfile); - } - }); - - return familyFiles; + public static Set getHFileNames(final Configuration conf, final FileSystem fs, + final Path snapshotDir) throws IOException { + SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); + return getHFileNames(conf, fs, snapshotDir, desc); } /** * Returns the store file names in the snapshot. * + * @param conf The current {@link Configuration} instance. * @param fs {@link FileSystem} * @param snapshotDir {@link Path} to the Snapshot directory + * @param snapshotDesc the {@link SnapshotDescription} of the snapshot to inspect * @throws IOException if an error occurred while scanning the directory * @return the names of hfiles in the specified snaphot */ - public static Set getHFileNames(final FileSystem fs, final Path snapshotDir) + private static Set getHFileNames(final Configuration conf, final FileSystem fs, + final Path snapshotDir, final SnapshotDescription snapshotDesc) throws IOException { final Set names = new HashSet(); - visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() { - public void storeFile (final String region, final String family, final String hfile) - throws IOException { + visitTableStoreFiles(conf, fs, snapshotDir, snapshotDesc, new StoreFileVisitor() { + @Override + public void storeFile(final HRegionInfo regionInfo, final String family, + final SnapshotRegionManifest.StoreFile storeFile) throws IOException { + String hfile = storeFile.getName(); if (HFileLink.isHFileLink(hfile)) { names.add(HFileLink.getReferencedHFileName(hfile)); } else { @@ -266,6 +342,7 @@ public final class SnapshotReferenceUtil throws IOException { final Set names = new HashSet(); visitLogFiles(fs, snapshotDir, new FSVisitor.LogFileVisitor() { + @Override public void logFile (final String server, final String logfile) throws IOException { names.add(logfile); } Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSVisitor.java URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSVisitor.java?rev=1593139&r1=1593138&r2=1593139&view=diff ============================================================================== --- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSVisitor.java (original) +++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSVisitor.java Wed May 7 21:28:12 2014 @@ -150,7 +150,7 @@ public final class FSVisitor { final FSVisitor.RecoveredEditsVisitor visitor) throws IOException { FileStatus[] regions = FSUtils.listStatus(fs, tableDir, new FSUtils.RegionDirFilter(fs)); if (regions == null) { - LOG.info("No regions under directory:" + tableDir); + LOG.info("No recoveredEdits regions under directory:" + tableDir); return; } Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java?rev=1593139&r1=1593138&r2=1593139&view=diff ============================================================================== --- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java (original) +++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java Wed May 7 21:28:12 2014 @@ -20,9 +20,7 @@ package org.apache.hadoop.hbase.client; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; -import java.util.HashSet; import java.util.List; -import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -39,9 +37,9 @@ import org.apache.hadoop.hbase.protobuf. import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy; import org.apache.hadoop.hbase.snapshot.SnapshotCreationException; import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils; +import org.apache.hadoop.hbase.snapshot.SnapshotManifestV1; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSUtils; -import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -182,15 +180,6 @@ public class TestSnapshotFromClient { HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME); UTIL.loadTable(table, TEST_FAM, false); - // get the name of all the regionservers hosting the snapshotted table - Set snapshotServers = new HashSet(); - List servers = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads(); - for (RegionServerThread server : servers) { - if (server.getRegionServer().getOnlineRegions(TABLE_NAME).size() > 0) { - snapshotServers.add(server.getRegionServer().getServerName().toString()); - } - } - LOG.debug("FS state before disable:"); FSUtils.logFileSystemState(UTIL.getTestFileSystem(), FSUtils.getRootDir(UTIL.getConfiguration()), LOG); @@ -203,8 +192,16 @@ public class TestSnapshotFromClient { FSUtils.getRootDir(UTIL.getConfiguration()), LOG); // take a snapshot of the disabled table - byte[] snapshot = Bytes.toBytes("offlineTableSnapshot"); - admin.snapshot(snapshot, TABLE_NAME); + final String SNAPSHOT_NAME = "offlineTableSnapshot"; + byte[] snapshot = Bytes.toBytes(SNAPSHOT_NAME); + + SnapshotDescription desc = SnapshotDescription.newBuilder() + .setType(SnapshotDescription.Type.DISABLED) + .setTable(STRING_TABLE_NAME) + .setName(SNAPSHOT_NAME) + .setVersion(SnapshotManifestV1.DESCRIPTOR_VERSION) + .build(); + admin.snapshot(desc); LOG.debug("Snapshot completed."); // make sure we have the snapshot @@ -219,7 +216,7 @@ public class TestSnapshotFromClient { FSUtils.getRootDir(UTIL.getConfiguration()), LOG); SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir, - admin, fs, false, new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), snapshotServers); + admin, fs); admin.deleteSnapshot(snapshot); snapshots = admin.listSnapshots(); @@ -263,15 +260,6 @@ public class TestSnapshotFromClient { // make sure we don't fail on listing snapshots SnapshotTestingUtils.assertNoSnapshots(admin); - // get the name of all the regionservers hosting the snapshotted table - Set snapshotServers = new HashSet(); - List servers = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads(); - for (RegionServerThread server : servers) { - if (server.getRegionServer().getOnlineRegions(TABLE_NAME).size() > 0) { - snapshotServers.add(server.getRegionServer().getServerName().toString()); - } - } - LOG.debug("FS state before disable:"); FSUtils.logFileSystemState(UTIL.getTestFileSystem(), FSUtils.getRootDir(UTIL.getConfiguration()), LOG); @@ -299,8 +287,8 @@ public class TestSnapshotFromClient { List emptyCfs = Lists.newArrayList(TEST_FAM); // no file in the region List nonEmptyCfs = Lists.newArrayList(); - SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, nonEmptyCfs, emptyCfs, rootDir, - admin, fs, false, new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), snapshotServers); + SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, nonEmptyCfs, emptyCfs, + rootDir, admin, fs); admin.deleteSnapshot(snapshot); snapshots = admin.listSnapshots(); Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java?rev=1593139&r1=1593138&r2=1593139&view=diff ============================================================================== --- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java (original) +++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java Wed May 7 21:28:12 2014 @@ -29,13 +29,13 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellScanner; import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.LargeTests; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.mapreduce.TestTableSnapshotInputFormat; import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSUtils; import org.junit.After; import org.junit.Assert; import org.junit.Test; @@ -94,7 +94,7 @@ public class TestTableSnapshotScanner { HTable table = new HTable(util.getConfiguration(), tableName); util.loadTable(table, FAMILIES); - Path rootDir = new Path(util.getConfiguration().get(HConstants.HBASE_DIR)); + Path rootDir = FSUtils.getRootDir(util.getConfiguration()); FileSystem fs = rootDir.getFileSystem(util.getConfiguration()); SnapshotTestingUtils.createSnapshotAndValidate(admin, tableName, @@ -124,7 +124,7 @@ public class TestTableSnapshotScanner { testScanner(UTIL, "testWithMultiRegion", 20, true); } - private void testScanner(HBaseTestingUtility util, String snapshotName, int numRegions, + private void testScanner(HBaseTestingUtility util, String snapshotName, int numRegions, boolean shutdownCluster) throws Exception { setupCluster(); TableName tableName = TableName.valueOf("testScanner"); @@ -138,7 +138,7 @@ public class TestTableSnapshotScanner { Path restoreDir = util.getDataTestDirOnTestFS(snapshotName); Scan scan = new Scan(bbb, yyy); // limit the scan - TableSnapshotScanner scanner = new TableSnapshotScanner(UTIL.getConfiguration(), restoreDir, + TableSnapshotScanner scanner = new TableSnapshotScanner(UTIL.getConfiguration(), restoreDir, snapshotName, scan); verifyScanner(scanner, bbb, yyy); @@ -155,7 +155,7 @@ public class TestTableSnapshotScanner { private void verifyScanner(ResultScanner scanner, byte[] startRow, byte[] stopRow) throws IOException, InterruptedException { - HBaseTestingUtility.SeenRowTracker rowTracker = + HBaseTestingUtility.SeenRowTracker rowTracker = new HBaseTestingUtility.SeenRowTracker(startRow, stopRow); while (true) { Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java?rev=1593139&r1=1593138&r2=1593139&view=diff ============================================================================== --- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java (original) +++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java Wed May 7 21:28:12 2014 @@ -32,7 +32,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellScanner; -import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HDFSBlocksDistribution; @@ -44,12 +43,12 @@ import org.apache.hadoop.hbase.client.Re import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.io.hfile.BlockCache; -import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.LruBlockCache; import org.apache.hadoop.hbase.mapreduce.TableSnapshotInputFormat.TableSnapshotRegionSplit; import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapreduce.InputSplit; import org.apache.hadoop.mapreduce.Job; @@ -59,7 +58,6 @@ import org.apache.hadoop.mapreduce.TaskA import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat; import org.junit.After; import org.junit.Assert; -import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -125,17 +123,17 @@ public class TestTableSnapshotInputForma Assert.assertEquals(Lists.newArrayList("h1"), tsif.getBestLocations(conf, blockDistribution)); blockDistribution.addHostsAndBlockWeight(new String[] {"h2"}, 2); - Assert.assertEquals(Lists.newArrayList("h1", "h2"), + Assert.assertEquals(Lists.newArrayList("h1", "h2"), tsif.getBestLocations(conf, blockDistribution)); blockDistribution.addHostsAndBlockWeight(new String[] {"h2"}, 3); - Assert.assertEquals(Lists.newArrayList("h2", "h1"), + Assert.assertEquals(Lists.newArrayList("h2", "h1"), tsif.getBestLocations(conf, blockDistribution)); blockDistribution.addHostsAndBlockWeight(new String[] {"h3"}, 6); blockDistribution.addHostsAndBlockWeight(new String[] {"h4"}, 9); - Assert.assertEquals(Lists.newArrayList("h2", "h3", "h4", "h1"), + Assert.assertEquals(Lists.newArrayList("h2", "h3", "h4", "h1"), tsif.getBestLocations(conf, blockDistribution)); } @@ -156,7 +154,7 @@ public class TestTableSnapshotInputForma public static class TestTableSnapshotReducer extends Reducer { - HBaseTestingUtility.SeenRowTracker rowTracker = + HBaseTestingUtility.SeenRowTracker rowTracker = new HBaseTestingUtility.SeenRowTracker(bbb, yyy); @Override protected void reduce(ImmutableBytesWritable key, Iterable values, @@ -191,7 +189,7 @@ public class TestTableSnapshotInputForma HTable table = new HTable(util.getConfiguration(), tableName); util.loadTable(table, FAMILIES); - Path rootDir = new Path(util.getConfiguration().get(HConstants.HBASE_DIR)); + Path rootDir = FSUtils.getRootDir(util.getConfiguration()); FileSystem fs = rootDir.getFileSystem(util.getConfiguration()); SnapshotTestingUtils.createSnapshotAndValidate(admin, tableName, @@ -250,7 +248,7 @@ public class TestTableSnapshotInputForma testWithMockedMapReduce(UTIL, "testWithMockedMapReduceMultiRegion", 10, 8); } - public void testWithMockedMapReduce(HBaseTestingUtility util, String snapshotName, + public void testWithMockedMapReduce(HBaseTestingUtility util, String snapshotName, int numRegions, int expectedNumSplits) throws Exception { setupCluster(); TableName tableName = TableName.valueOf("testWithMockedMapReduce"); @@ -282,7 +280,7 @@ public class TestTableSnapshotInputForma Assert.assertEquals(expectedNumSplits, splits.size()); - HBaseTestingUtility.SeenRowTracker rowTracker = + HBaseTestingUtility.SeenRowTracker rowTracker = new HBaseTestingUtility.SeenRowTracker(startRow, stopRow); for (int i = 0; i < splits.size(); i++) { @@ -293,7 +291,7 @@ public class TestTableSnapshotInputForma // validate record reader TaskAttemptContext taskAttemptContext = mock(TaskAttemptContext.class); when(taskAttemptContext.getConfiguration()).thenReturn(job.getConfiguration()); - RecordReader rr = + RecordReader rr = tsif.createRecordReader(split, taskAttemptContext); rr.initialize(split, taskAttemptContext); @@ -311,7 +309,7 @@ public class TestTableSnapshotInputForma rowTracker.validate(); } - public static void verifyRowFromMap(ImmutableBytesWritable key, Result result) + public static void verifyRowFromMap(ImmutableBytesWritable key, Result result) throws IOException { byte[] row = key.get(); CellScanner scanner = result.cellScanner(); @@ -363,7 +361,7 @@ public class TestTableSnapshotInputForma // this is also called by the IntegrationTestTableSnapshotInputFormat public static void doTestWithMapReduce(HBaseTestingUtility util, TableName tableName, - String snapshotName, Path tableDir, int numRegions, int expectedNumSplits, + String snapshotName, Path tableDir, int numRegions, int expectedNumSplits, boolean shutdownCluster) throws Exception { //create the table and snapshot @@ -379,7 +377,7 @@ public class TestTableSnapshotInputForma Scan scan = new Scan(bbb, yyy); // limit the scan job.setJarByClass(util.getClass()); - TableMapReduceUtil.addDependencyJars(job.getConfiguration(), + TableMapReduceUtil.addDependencyJars(job.getConfiguration(), TestTableSnapshotInputFormat.class); TableMapReduceUtil.initTableSnapshotMapperJob(snapshotName, Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java?rev=1593139&r1=1593138&r2=1593139&view=diff ============================================================================== --- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java (original) +++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java Wed May 7 21:28:12 2014 @@ -26,6 +26,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -52,6 +53,7 @@ import org.apache.hadoop.hbase.protobuf. import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; +import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil; import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils; import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException; import org.apache.hadoop.hbase.util.Bytes; @@ -330,19 +332,20 @@ public class TestSnapshotFromMaster { // get the snapshot files for the table Path snapshotTable = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); - Path[] snapshotHFiles = SnapshotTestingUtils.listHFiles(fs, snapshotTable); + Set snapshotHFiles = SnapshotReferenceUtil.getHFileNames( + UTIL.getConfiguration(), fs, snapshotTable); // check that the files in the archive contain the ones that we need for the snapshot LOG.debug("Have snapshot hfiles:"); - for (Path file : snapshotHFiles) { - LOG.debug(file); + for (String fileName : snapshotHFiles) { + LOG.debug(fileName); } // get the archived files for the table Collection files = getArchivedHFiles(archiveDir, rootDir, fs, TABLE_NAME); // and make sure that there is a proper subset - for (Path file : snapshotHFiles) { - assertTrue("Archived hfiles " + files + " is missing snapshot file:" + file, - files.contains(file.getName())); + for (String fileName : snapshotHFiles) { + assertTrue("Archived hfiles " + files + " is missing snapshot file:" + fileName, + files.contains(fileName)); } // delete the existing snapshot Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java?rev=1593139&r1=1593138&r2=1593139&view=diff ============================================================================== --- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java (original) +++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java Wed May 7 21:28:12 2014 @@ -22,7 +22,9 @@ import static org.junit.Assert.assertTru import java.io.IOException; import java.util.Collection; +import java.util.ArrayList; import java.util.HashSet; +import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -31,9 +33,10 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.MediumTests; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.regionserver.wal.HLogUtil; import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil; -import org.apache.hadoop.hbase.snapshot.TakeSnapshotUtils; +import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils.SnapshotMock; import org.apache.hadoop.hbase.util.FSUtils; import org.junit.After; import org.junit.AfterClass; @@ -49,6 +52,7 @@ public class TestSnapshotFileCache { private static final Log LOG = LogFactory.getLog(TestSnapshotFileCache.class); private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private static long sequenceId = 0; private static FileSystem fs; private static Path rootDir; @@ -72,86 +76,17 @@ public class TestSnapshotFileCache { } @Test(timeout = 10000000) - public void testLoadAndDelete() throws Exception { + public void testLoadAndDelete() throws IOException { // don't refresh the cache unless we tell it to long period = Long.MAX_VALUE; - Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir); SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, "test-snapshot-file-cache-refresh", new SnapshotFiles()); - Path snapshot = new Path(snapshotDir, "snapshot"); - Path region = new Path(snapshot, "7e91021"); - Path family = new Path(region, "fam"); - Path file1 = new Path(family, "file1"); - Path file2 = new Path(family, "file2"); - - // create two hfiles under the snapshot - fs.createNewFile(file1); - fs.createNewFile(file2); + createAndTestSnapshotV1(cache, "snapshot1a", false, true); + createAndTestSnapshotV1(cache, "snapshot1b", true, true); - FSUtils.logFileSystemState(fs, rootDir, LOG); - - // then make sure the cache finds them - assertTrue("Cache didn't find:" + file1, cache.contains(file1.getName())); - assertTrue("Cache didn't find:" + file2, cache.contains(file2.getName())); - String not = "file-shouldn't-be-found"; - assertFalse("Cache found '" + not + "', but it shouldn't have.", cache.contains(not)); - - // make sure we get a little bit of separation in the modification times - // its okay if we sleep a little longer (b/c of GC pause), as long as we sleep a little - Thread.sleep(10); - - LOG.debug("Deleting snapshot."); - // then delete the snapshot and make sure that we can still find the files - if (!fs.delete(snapshot, true)) { - throw new IOException("Couldn't delete " + snapshot + " for an unknown reason."); - } - FSUtils.logFileSystemState(fs, rootDir, LOG); - - - LOG.debug("Checking to see if file is deleted."); - assertTrue("Cache didn't find:" + file1, cache.contains(file1.getName())); - assertTrue("Cache didn't find:" + file2, cache.contains(file2.getName())); - - // then trigger a refresh - cache.triggerCacheRefreshForTesting(); - // and not it shouldn't find those files - assertFalse("Cache found '" + file1 + "', but it shouldn't have.", - cache.contains(file1.getName())); - assertFalse("Cache found '" + file2 + "', but it shouldn't have.", - cache.contains(file2.getName())); - - fs.delete(snapshotDir, true); - } - - @Test - public void testLoadsTmpDir() throws Exception { - // don't refresh the cache unless we tell it to - long period = Long.MAX_VALUE; - Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir); - SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, - "test-snapshot-file-cache-refresh", new SnapshotFiles()); - - // create a file in a 'completed' snapshot - Path snapshot = new Path(snapshotDir, "snapshot"); - Path region = new Path(snapshot, "7e91021"); - Path family = new Path(region, "fam"); - Path file1 = new Path(family, "file1"); - fs.createNewFile(file1); - - // create an 'in progress' snapshot - SnapshotDescription desc = SnapshotDescription.newBuilder().setName("working").build(); - snapshot = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir); - region = new Path(snapshot, "7e91021"); - family = new Path(region, "fam"); - Path file2 = new Path(family, "file2"); - fs.createNewFile(file2); - - FSUtils.logFileSystemState(fs, rootDir, LOG); - - // then make sure the cache finds both files - assertTrue("Cache didn't find:" + file1, cache.contains(file1.getName())); - assertTrue("Cache didn't find:" + file2, cache.contains(file2.getName())); + createAndTestSnapshotV2(cache, "snapshot2a", false, true); + createAndTestSnapshotV2(cache, "snapshot2b", true, true); } @Test @@ -168,14 +103,14 @@ public class TestSnapshotFileCache { }); // create a file in a 'completed' snapshot - Path snapshot = new Path(snapshotDir, "snapshot"); - Path region = new Path(snapshot, "7e91021"); - Path family = new Path(region, "fam"); - Path file1 = new Path(family, "file1"); + SnapshotDescription desc = SnapshotDescription.newBuilder().setName("snapshot").build(); + Path snapshot = SnapshotDescriptionUtils.getCompletedSnapshotDir(desc, rootDir); + SnapshotDescriptionUtils.writeSnapshotInfo(desc, snapshot, fs); + Path file1 = new Path(new Path(new Path(snapshot, "7e91021"), "fam"), "file1"); fs.createNewFile(file1); // and another file in the logs directory - Path logs = TakeSnapshotUtils.getSnapshotHLogsDir(snapshot, "server"); + Path logs = getSnapshotHLogsDir(snapshot, "server"); Path log = new Path(logs, "me.hbase.com%2C58939%2C1350424310315.1350424315552"); fs.createNewFile(log); @@ -187,65 +122,117 @@ public class TestSnapshotFileCache { assertTrue("Cache didn't find:" + log, cache.contains(log.getName())); } + /** + * Get the log directory for a specific snapshot + * @param snapshotDir directory where the specific snapshot will be store + * @param serverName name of the parent regionserver for the log files + * @return path to the log home directory for the archive files. + */ + public static Path getSnapshotHLogsDir(Path snapshotDir, String serverName) { + return new Path(snapshotDir, HLogUtil.getHLogDirectoryName(serverName)); + } + @Test public void testReloadModifiedDirectory() throws IOException { // don't refresh the cache unless we tell it to long period = Long.MAX_VALUE; - Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir); SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, "test-snapshot-file-cache-refresh", new SnapshotFiles()); - Path snapshot = new Path(snapshotDir, "snapshot"); - Path region = new Path(snapshot, "7e91021"); - Path family = new Path(region, "fam"); - Path file1 = new Path(family, "file1"); - Path file2 = new Path(family, "file2"); - - // create two hfiles under the snapshot - fs.createNewFile(file1); - fs.createNewFile(file2); - - FSUtils.logFileSystemState(fs, rootDir, LOG); - - assertTrue("Cache didn't find " + file1, cache.contains(file1.getName())); - + createAndTestSnapshotV1(cache, "snapshot1", false, true); // now delete the snapshot and add a file with a different name - fs.delete(snapshot, true); - Path file3 = new Path(family, "new_file"); - fs.createNewFile(file3); + createAndTestSnapshotV1(cache, "snapshot1", false, false); - FSUtils.logFileSystemState(fs, rootDir, LOG); - assertTrue("Cache didn't find new file:" + file3, cache.contains(file3.getName())); + createAndTestSnapshotV2(cache, "snapshot2", false, true); + // now delete the snapshot and add a file with a different name + createAndTestSnapshotV2(cache, "snapshot2", false, false); } @Test public void testSnapshotTempDirReload() throws IOException { long period = Long.MAX_VALUE; // This doesn't refresh cache until we invoke it explicitly - Path snapshotDir = new Path(SnapshotDescriptionUtils.getSnapshotsDir(rootDir), - SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME); SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, "test-snapshot-file-cache-refresh", new SnapshotFiles()); - // Add a new snapshot - Path snapshot1 = new Path(snapshotDir, "snapshot1"); - Path file1 = new Path(new Path(new Path(snapshot1, "7e91021"), "fam"), "file1"); - fs.createNewFile(file1); - assertTrue(cache.contains(file1.getName())); + // Add a new non-tmp snapshot + createAndTestSnapshotV1(cache, "snapshot0v1", false, false); + createAndTestSnapshotV1(cache, "snapshot0v2", false, false); + + // Add a new tmp snapshot + createAndTestSnapshotV2(cache, "snapshot1", true, false); - // Add another snapshot - Path snapshot2 = new Path(snapshotDir, "snapshot2"); - Path file2 = new Path(new Path(new Path(snapshot2, "7e91021"), "fam2"), "file2"); - fs.createNewFile(file2); - assertTrue(cache.contains(file2.getName())); + // Add another tmp snapshot + createAndTestSnapshotV2(cache, "snapshot2", true, false); } class SnapshotFiles implements SnapshotFileCache.SnapshotFileInspector { public Collection filesUnderSnapshot(final Path snapshotDir) throws IOException { Collection files = new HashSet(); files.addAll(SnapshotReferenceUtil.getHLogNames(fs, snapshotDir)); - files.addAll(SnapshotReferenceUtil.getHFileNames(fs, snapshotDir)); + files.addAll(SnapshotReferenceUtil.getHFileNames(UTIL.getConfiguration(), fs, snapshotDir)); return files; } }; + + private void createAndTestSnapshotV1(final SnapshotFileCache cache, final String name, + final boolean tmp, final boolean removeOnExit) throws IOException { + SnapshotMock snapshotMock = new SnapshotMock(UTIL.getConfiguration(), fs, rootDir); + SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV1(name); + createAndTestSnapshot(cache, builder, tmp, removeOnExit); + } + + private void createAndTestSnapshotV2(final SnapshotFileCache cache, final String name, + final boolean tmp, final boolean removeOnExit) throws IOException { + SnapshotMock snapshotMock = new SnapshotMock(UTIL.getConfiguration(), fs, rootDir); + SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2(name); + createAndTestSnapshot(cache, builder, tmp, removeOnExit); + } + + private void createAndTestSnapshot(final SnapshotFileCache cache, + final SnapshotMock.SnapshotBuilder builder, + final boolean tmp, final boolean removeOnExit) throws IOException { + List files = new ArrayList(); + for (int i = 0; i < 3; ++i) { + for (Path filePath: builder.addRegion()) { + String fileName = filePath.getName(); + if (tmp) { + // We should be able to find all the files while the snapshot creation is in-progress + FSUtils.logFileSystemState(fs, rootDir, LOG); + assertTrue("Cache didn't find " + fileName, cache.contains(fileName)); + } + files.add(fileName); + } + } + + // Finalize the snapshot + if (!tmp) { + builder.commit(); + } + + // Make sure that all files are still present + for (String fileName: files) { + assertTrue("Cache didn't find " + fileName, cache.contains(fileName)); + } + + FSUtils.logFileSystemState(fs, rootDir, LOG); + if (removeOnExit) { + LOG.debug("Deleting snapshot."); + fs.delete(builder.getSnapshotsDir(), true); + FSUtils.logFileSystemState(fs, rootDir, LOG); + + // The files should be in cache until next refresh + for (String fileName: files) { + assertTrue("Cache didn't find " + fileName, cache.contains(fileName)); + } + + // then trigger a refresh + cache.triggerCacheRefreshForTesting(); + // and not it shouldn't find those files + for (String fileName: files) { + assertFalse("Cache found '" + fileName + "', but it shouldn't have.", + cache.contains(fileName)); + } + } + } } Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java?rev=1593139&r1=1593138&r2=1593139&view=diff ============================================================================== --- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java (original) +++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java Wed May 7 21:28:12 2014 @@ -22,14 +22,18 @@ import static org.junit.Assert.assertFal import static org.junit.Assert.assertTrue; import java.io.IOException; +import java.util.Arrays; import java.util.ArrayList; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.TreeSet; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseTestingUtility; @@ -43,11 +47,13 @@ import org.apache.hadoop.hbase.client.Du import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher; import org.apache.hadoop.hbase.io.HFileLink; import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.master.MasterFileSystem; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneResponse; import org.apache.hadoop.hbase.regionserver.HRegion; @@ -56,6 +62,7 @@ import org.apache.hadoop.hbase.regionser import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSTableDescriptors; import org.apache.hadoop.hbase.util.FSVisitor; +import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.MD5Hash; import org.junit.Assert; @@ -144,14 +151,12 @@ public class SnapshotTestingUtils { */ public static void confirmSnapshotValid( SnapshotDescription snapshotDescriptor, TableName tableName, - byte[] testFamily, Path rootDir, HBaseAdmin admin, FileSystem fs, - boolean requireLogs, Path logsDir, Set snapshotServers) + byte[] testFamily, Path rootDir, HBaseAdmin admin, FileSystem fs) throws IOException { ArrayList nonEmptyTestFamilies = new ArrayList(1); nonEmptyTestFamilies.add(testFamily); confirmSnapshotValid(snapshotDescriptor, tableName, - nonEmptyTestFamilies, null, rootDir, admin, fs, requireLogs, - logsDir, snapshotServers); + nonEmptyTestFamilies, null, rootDir, admin, fs); } /** @@ -159,14 +164,12 @@ public class SnapshotTestingUtils { */ public static void confirmEmptySnapshotValid( SnapshotDescription snapshotDescriptor, TableName tableName, - byte[] testFamily, Path rootDir, HBaseAdmin admin, FileSystem fs, - boolean requireLogs, Path logsDir, Set snapshotServers) + byte[] testFamily, Path rootDir, HBaseAdmin admin, FileSystem fs) throws IOException { ArrayList emptyTestFamilies = new ArrayList(1); emptyTestFamilies.add(testFamily); confirmSnapshotValid(snapshotDescriptor, tableName, - null, emptyTestFamilies, rootDir, admin, fs, requireLogs, - logsDir, snapshotServers); + null, emptyTestFamilies, rootDir, admin, fs); } /** @@ -178,45 +181,31 @@ public class SnapshotTestingUtils { public static void confirmSnapshotValid( SnapshotDescription snapshotDescriptor, TableName tableName, List nonEmptyTestFamilies, List emptyTestFamilies, - Path rootDir, HBaseAdmin admin, FileSystem fs, boolean requireLogs, - Path logsDir, Set snapshotServers) throws IOException { + Path rootDir, HBaseAdmin admin, FileSystem fs) throws IOException { + final Configuration conf = admin.getConfiguration(); + // check snapshot dir Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir( snapshotDescriptor, rootDir); assertTrue(fs.exists(snapshotDir)); - // check snapshot info - Path snapshotinfo = new Path(snapshotDir, SnapshotDescriptionUtils.SNAPSHOTINFO_FILE); - assertTrue(fs.exists(snapshotinfo)); - - // check the logs dir - if (requireLogs) { - TakeSnapshotUtils.verifyAllLogsGotReferenced(fs, logsDir, - snapshotServers, snapshotDescriptor, new Path(snapshotDir, - HConstants.HREGION_LOGDIR_NAME)); - } - - // check the table info - HTableDescriptor desc = FSTableDescriptors.getTableDescriptorFromFs(fs, rootDir, tableName); - HTableDescriptor snapshotDesc = FSTableDescriptors.getTableDescriptorFromFs(fs, snapshotDir); - assertEquals(desc, snapshotDesc); + SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); // Extract regions and families with store files - final Set snapshotRegions = new HashSet(); final Set snapshotFamilies = new TreeSet(Bytes.BYTES_COMPARATOR); - FSVisitor.visitRegions(fs, snapshotDir, new FSVisitor.RegionVisitor() { - @Override - public void region(String region) throws IOException { - snapshotRegions.add(region); - } - }); - FSVisitor.visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() { - @Override - public void storeFile(final String region, final String family, final String hfileName) - throws IOException { - snapshotFamilies.add(Bytes.toBytes(family)); - } - }); + + SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, desc); + Map regionManifests = manifest.getRegionManifestsMap(); + for (SnapshotRegionManifest regionManifest: regionManifests.values()) { + SnapshotReferenceUtil.visitRegionStoreFiles(regionManifest, + new SnapshotReferenceUtil.StoreFileVisitor() { + @Override + public void storeFile(final HRegionInfo regionInfo, final String family, + final SnapshotRegionManifest.StoreFile storeFile) throws IOException { + snapshotFamilies.add(Bytes.toBytes(family)); + } + }); + } // Verify that there are store files in the specified families if (nonEmptyTestFamilies != null) { @@ -234,16 +223,12 @@ public class SnapshotTestingUtils { // check the region snapshot for all the regions List regions = admin.getTableRegions(tableName); - assertEquals(regions.size(), snapshotRegions.size()); + assertEquals(regions.size(), regionManifests.size()); - // Verify Regions + // Verify Regions (redundant check, see MasterSnapshotVerifier) for (HRegionInfo info : regions) { String regionName = info.getEncodedName(); - assertTrue(snapshotRegions.contains(regionName)); - - Path regionDir = new Path(snapshotDir, regionName); - HRegionInfo snapshotRegionInfo = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir); - assertEquals(info, snapshotRegionInfo); + assertTrue(regionManifests.containsKey(regionName)); } } @@ -351,6 +336,17 @@ public class SnapshotTestingUtils { return hfiles.toArray(new Path[hfiles.size()]); } + public static String[] listHFileNames(final FileSystem fs, final Path tableDir) + throws IOException { + Path[] files = listHFiles(fs, tableDir); + String[] names = new String[files.length]; + for (int i = 0; i < files.length; ++i) { + names[i] = files[i].getName(); + } + Arrays.sort(names); + return names; + } + /** * Take a snapshot of the specified table and verify that the given family is * not empty. Note that this will leave the table disabled @@ -391,8 +387,7 @@ public class SnapshotTestingUtils { } SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), tableName, nonEmptyFamilyNames, - emptyFamilyNames, rootDir, admin, fs, false, - new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), null); + emptyFamilyNames, rootDir, admin, fs); } /** @@ -414,11 +409,15 @@ public class SnapshotTestingUtils { final TableName table = TableName.valueOf(snapshotDesc.getTable()); final ArrayList corruptedFiles = new ArrayList(); - SnapshotReferenceUtil.visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() { + final Configuration conf = util.getConfiguration(); + SnapshotReferenceUtil.visitTableStoreFiles(conf, fs, snapshotDir, snapshotDesc, + new SnapshotReferenceUtil.StoreFileVisitor() { @Override - public void storeFile (final String region, final String family, final String hfile) - throws IOException { - HFileLink link = HFileLink.create(util.getConfiguration(), table, region, family, hfile); + public void storeFile(final HRegionInfo regionInfo, final String family, + final SnapshotRegionManifest.StoreFile storeFile) throws IOException { + String region = regionInfo.getEncodedName(); + String hfile = storeFile.getName(); + HFileLink link = HFileLink.create(conf, table, region, family, hfile); if (corruptedFiles.size() % 2 == 0) { fs.delete(link.getAvailablePath(fs), true); corruptedFiles.add(hfile); @@ -431,6 +430,191 @@ public class SnapshotTestingUtils { } // ========================================================================== + // Snapshot Mock + // ========================================================================== + public static class SnapshotMock { + private final static String TEST_FAMILY = "cf"; + public final static int TEST_NUM_REGIONS = 4; + + private final Configuration conf; + private final FileSystem fs; + private final Path rootDir; + + static class RegionData { + public HRegionInfo hri; + public Path tableDir; + public Path[] files; + + public RegionData(final Path tableDir, final HRegionInfo hri, final int nfiles) { + this.tableDir = tableDir; + this.hri = hri; + this.files = new Path[nfiles]; + } + } + + public static class SnapshotBuilder { + private final RegionData[] tableRegions; + private final SnapshotDescription desc; + private final HTableDescriptor htd; + private final Configuration conf; + private final FileSystem fs; + private final Path rootDir; + private Path snapshotDir; + private int snapshotted = 0; + + public SnapshotBuilder(final Configuration conf, final FileSystem fs, + final Path rootDir, final HTableDescriptor htd, + final SnapshotDescription desc, final RegionData[] tableRegions) + throws IOException { + this.fs = fs; + this.conf = conf; + this.rootDir = rootDir; + this.htd = htd; + this.desc = desc; + this.tableRegions = tableRegions; + this.snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir); + new FSTableDescriptors(conf) + .createTableDescriptorForTableDirectory(snapshotDir, htd, false); + } + + public HTableDescriptor getTableDescriptor() { + return this.htd; + } + + public SnapshotDescription getSnapshotDescription() { + return this.desc; + } + + public Path getSnapshotsDir() { + return this.snapshotDir; + } + + public Path[] addRegion() throws IOException { + return addRegion(desc); + } + + public Path[] addRegionV1() throws IOException { + return addRegion(desc.toBuilder() + .setVersion(SnapshotManifestV1.DESCRIPTOR_VERSION) + .build()); + } + + public Path[] addRegionV2() throws IOException { + return addRegion(desc.toBuilder() + .setVersion(SnapshotManifestV2.DESCRIPTOR_VERSION) + .build()); + } + + private Path[] addRegion(final SnapshotDescription desc) throws IOException { + if (this.snapshotted == tableRegions.length) { + throw new UnsupportedOperationException("No more regions in the table"); + } + + RegionData regionData = tableRegions[this.snapshotted++]; + ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName()); + SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor); + manifest.addRegion(regionData.tableDir, regionData.hri); + return regionData.files; + } + + public Path commit() throws IOException { + ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName()); + SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor); + manifest.addTableDescriptor(htd); + manifest.consolidate(); + SnapshotDescriptionUtils.completeSnapshot(desc, rootDir, snapshotDir, fs); + snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(desc, rootDir); + return snapshotDir; + } + } + + public SnapshotMock(final Configuration conf, final FileSystem fs, final Path rootDir) { + this.fs = fs; + this.conf = conf; + this.rootDir = rootDir; + } + + public SnapshotBuilder createSnapshotV1(final String snapshotName) throws IOException { + return createSnapshot(snapshotName, SnapshotManifestV1.DESCRIPTOR_VERSION); + } + + public SnapshotBuilder createSnapshotV2(final String snapshotName) throws IOException { + return createSnapshot(snapshotName, SnapshotManifestV2.DESCRIPTOR_VERSION); + } + + private SnapshotBuilder createSnapshot(final String snapshotName, final int version) + throws IOException { + HTableDescriptor htd = createHtd(snapshotName); + htd.addFamily(new HColumnDescriptor(TEST_FAMILY)); + + RegionData[] regions = createTable(htd, TEST_NUM_REGIONS); + + SnapshotDescription desc = SnapshotDescription.newBuilder() + .setTable(htd.getNameAsString()) + .setName(snapshotName) + .setVersion(version) + .build(); + + Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir); + SnapshotDescriptionUtils.writeSnapshotInfo(desc, workingDir, fs); + return new SnapshotBuilder(conf, fs, rootDir, htd, desc, regions); + } + + public HTableDescriptor createHtd(final String tableName) { + HTableDescriptor htd = new HTableDescriptor(tableName); + htd.addFamily(new HColumnDescriptor(TEST_FAMILY)); + return htd; + } + + private RegionData[] createTable(final HTableDescriptor htd, final int nregions) + throws IOException { + Path tableDir = FSUtils.getTableDir(rootDir, htd.getTableName()); + new FSTableDescriptors(conf).createTableDescriptorForTableDirectory(tableDir, htd, false); + + assertTrue(nregions % 2 == 0); + RegionData[] regions = new RegionData[nregions]; + for (int i = 0; i < regions.length; i += 2) { + byte[] startKey = Bytes.toBytes(0 + i * 2); + byte[] endKey = Bytes.toBytes(1 + i * 2); + + // First region, simple with one plain hfile. + HRegionInfo hri = new HRegionInfo(htd.getTableName(), startKey, endKey); + HRegionFileSystem rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri); + regions[i] = new RegionData(tableDir, hri, 3); + for (int j = 0; j < regions[i].files.length; ++j) { + Path storeFile = createStoreFile(rfs.createTempName()); + regions[i].files[j] = rfs.commitStoreFile(TEST_FAMILY, storeFile); + } + + // Second region, used to test the split case. + // This region contains a reference to the hfile in the first region. + startKey = Bytes.toBytes(2 + i * 2); + endKey = Bytes.toBytes(3 + i * 2); + hri = new HRegionInfo(htd.getTableName()); + rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri); + regions[i+1] = new RegionData(tableDir, hri, regions[i].files.length); + for (int j = 0; j < regions[i].files.length; ++j) { + String refName = regions[i].files[j].getName() + '.' + regions[i].hri.getEncodedName(); + Path refFile = createStoreFile(new Path(rootDir, refName)); + regions[i+1].files[j] = rfs.commitStoreFile(TEST_FAMILY, refFile); + } + } + return regions; + } + + private Path createStoreFile(final Path storeFile) + throws IOException { + FSDataOutputStream out = fs.create(storeFile); + try { + out.write(Bytes.toBytes(storeFile.toString())); + } finally { + out.close(); + } + return storeFile; + } + } + + // ========================================================================== // Table Helpers // ========================================================================== public static void waitForTableToBeOnline(final HBaseTestingUtility util, Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java?rev=1593139&r1=1593138&r2=1593139&view=diff ============================================================================== --- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java (original) +++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java Wed May 7 21:28:12 2014 @@ -24,7 +24,6 @@ import static org.junit.Assert.assertTru import java.io.IOException; import java.net.URI; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -48,9 +47,12 @@ import org.apache.hadoop.hbase.client.HB import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo; +import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.FSTableDescriptors; import org.apache.hadoop.hbase.util.Pair; import org.junit.After; import org.junit.AfterClass; @@ -143,9 +145,13 @@ public class TestExportSnapshot { @Test public void testBalanceSplit() throws Exception { // Create a list of files - List> files = new ArrayList>(); + List> files = new ArrayList>(); for (long i = 0; i <= 20; i++) { - files.add(new Pair(new Path("file-" + i), i)); + SnapshotFileInfo fileInfo = SnapshotFileInfo.newBuilder() + .setType(SnapshotFileInfo.Type.HFILE) + .setHfile("file-" + i) + .build(); + files.add(new Pair(fileInfo, i)); } // Create 5 groups (total size 210) @@ -154,18 +160,26 @@ public class TestExportSnapshot { // group 2: 18, 13, 8, 3 (total size: 42) // group 3: 17, 12, 7, 4 (total size: 42) // group 4: 16, 11, 6, 5 (total size: 42) - List> splits = ExportSnapshot.getBalancedSplits(files, 5); + List> splits = ExportSnapshot.getBalancedSplits(files, 5); assertEquals(5, splits.size()); - assertEquals(Arrays.asList(new Path("file-20"), new Path("file-11"), - new Path("file-10"), new Path("file-1"), new Path("file-0")), splits.get(0)); - assertEquals(Arrays.asList(new Path("file-19"), new Path("file-12"), - new Path("file-9"), new Path("file-2")), splits.get(1)); - assertEquals(Arrays.asList(new Path("file-18"), new Path("file-13"), - new Path("file-8"), new Path("file-3")), splits.get(2)); - assertEquals(Arrays.asList(new Path("file-17"), new Path("file-14"), - new Path("file-7"), new Path("file-4")), splits.get(3)); - assertEquals(Arrays.asList(new Path("file-16"), new Path("file-15"), - new Path("file-6"), new Path("file-5")), splits.get(4)); + + String[] split0 = new String[] {"file-20", "file-11", "file-10", "file-1", "file-0"}; + verifyBalanceSplit(splits.get(0), split0); + String[] split1 = new String[] {"file-19", "file-12", "file-9", "file-2"}; + verifyBalanceSplit(splits.get(1), split1); + String[] split2 = new String[] {"file-18", "file-13", "file-8", "file-3"}; + verifyBalanceSplit(splits.get(2), split2); + String[] split3 = new String[] {"file-17", "file-14", "file-7", "file-4"}; + verifyBalanceSplit(splits.get(3), split3); + String[] split4 = new String[] {"file-16", "file-15", "file-6", "file-5"}; + verifyBalanceSplit(splits.get(4), split4); + } + + private void verifyBalanceSplit(final List split, final String[] expected) { + assertEquals(expected.length, split.size()); + for (int i = 0; i < expected.length; ++i) { + assertEquals(expected[i], split.get(i).getHfile()); + } } /** @@ -233,6 +247,11 @@ public class TestExportSnapshot { r1fs.commitStoreFile(TEST_FAMILY, storeFile); Path tableDir = FSUtils.getTableDir(archiveDir, tableWithRefsName); + HTableDescriptor htd = new HTableDescriptor(tableWithRefsName); + htd.addFamily(new HColumnDescriptor(TEST_FAMILY)); + new FSTableDescriptors(fs, rootDir) + .createTableDescriptorForTableDirectory(tableDir, htd, false); + Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); FileUtil.copy(fs, tableDir, fs, snapshotDir, false, conf); SnapshotDescriptionUtils.writeSnapshotInfo(sd, snapshotDir, fs); @@ -345,13 +364,14 @@ public class TestExportSnapshot { new Path(HConstants.SNAPSHOT_DIR_NAME, snapshotName)); final Path exportedArchive = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY); LOG.debug(listFiles(fs, exportedArchive, exportedArchive)); - SnapshotReferenceUtil.visitReferencedFiles(fs, exportedSnapshot, - new SnapshotReferenceUtil.FileVisitor() { - public void storeFile (final String region, final String family, final String hfile) - throws IOException { + SnapshotReferenceUtil.visitReferencedFiles(TEST_UTIL.getConfiguration(), fs, exportedSnapshot, + new SnapshotReferenceUtil.SnapshotVisitor() { + public void storeFile(final HRegionInfo regionInfo, final String family, + final SnapshotRegionManifest.StoreFile storeFile) throws IOException { + String hfile = storeFile.getName(); verifyNonEmptyFile(new Path(exportedArchive, new Path(FSUtils.getTableDir(new Path("./"), tableName), - new Path(region, new Path(family, hfile))))); + new Path(regionInfo.getEncodedName(), new Path(family, hfile))))); } public void recoveredEdits (final String region, final String logfile) Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java?rev=1593139&r1=1593138&r2=1593139&view=diff ============================================================================== --- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java (original) +++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java Wed May 7 21:28:12 2014 @@ -40,7 +40,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.LargeTests; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableNotFoundException; @@ -53,10 +52,8 @@ import org.apache.hadoop.hbase.master.HM import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy; -import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.FSTableDescriptors; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; import org.apache.log4j.Level; @@ -153,15 +150,6 @@ public class TestFlushSnapshotFromClient HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME); SnapshotTestingUtils.loadData(UTIL, table, DEFAULT_NUM_ROWS, TEST_FAM); - // get the name of all the regionservers hosting the snapshotted table - Set snapshotServers = new HashSet(); - List servers = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads(); - for (RegionServerThread server : servers) { - if (server.getRegionServer().getOnlineRegions(TABLE_NAME).size() > 0) { - snapshotServers.add(server.getRegionServer().getServerName().toString()); - } - } - LOG.debug("FS state before snapshot:"); FSUtils.logFileSystemState(UTIL.getTestFileSystem(), FSUtils.getRootDir(UTIL.getConfiguration()), LOG); @@ -184,7 +172,7 @@ public class TestFlushSnapshotFromClient FSUtils.getRootDir(UTIL.getConfiguration()), LOG); SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir, - admin, fs, false, new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), snapshotServers); + admin, fs); } @@ -202,15 +190,6 @@ public class TestFlushSnapshotFromClient HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME); SnapshotTestingUtils.loadData(UTIL, table, DEFAULT_NUM_ROWS, TEST_FAM); - // get the name of all the regionservers hosting the snapshotted table - Set snapshotServers = new HashSet(); - List servers = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads(); - for (RegionServerThread server : servers) { - if (server.getRegionServer().getOnlineRegions(TABLE_NAME).size() > 0) { - snapshotServers.add(server.getRegionServer().getServerName().toString()); - } - } - LOG.debug("FS state before snapshot:"); FSUtils.logFileSystemState(UTIL.getTestFileSystem(), FSUtils.getRootDir(UTIL.getConfiguration()), LOG); @@ -238,7 +217,7 @@ public class TestFlushSnapshotFromClient FSUtils.getRootDir(UTIL.getConfiguration()), LOG); SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir, - admin, fs, false, new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), snapshotServers); + admin, fs); } @Test (timeout=300000) @@ -399,46 +378,11 @@ public class TestFlushSnapshotFromClient SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM); String snapshotName = "flushSnapshotCreateListDestroy"; - // test creating the snapshot - admin.snapshot(snapshotName, STRING_TABLE_NAME, SnapshotDescription.Type.FLUSH); - logFSTree(FSUtils.getRootDir(UTIL.getConfiguration())); - - // make sure we only have 1 matching snapshot - List snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, - snapshotName, TABLE_NAME); - - // check the directory structure FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem(); Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir(); - Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshots.get(0), rootDir); - assertTrue(fs.exists(snapshotDir)); - FSUtils.logFileSystemState(UTIL.getTestFileSystem(), snapshotDir, LOG); - Path snapshotinfo = new Path(snapshotDir, SnapshotDescriptionUtils.SNAPSHOTINFO_FILE); - assertTrue(fs.exists(snapshotinfo)); - - // check the table info - HTableDescriptor desc = FSTableDescriptors.getTableDescriptorFromFs(fs, - rootDir, TABLE_NAME); - HTableDescriptor snapshotDesc = FSTableDescriptors.getTableDescriptorFromFs(fs, - new Path(SnapshotDescriptionUtils.getSnapshotsDir(rootDir), snapshotName)); - assertEquals(desc, snapshotDesc); - - // check the region snapshot for all the regions - List regions = admin.getTableRegions(TABLE_NAME); - assertTrue(regions.size() > 1); - for (HRegionInfo info : regions) { - String regionName = info.getEncodedName(); - Path regionDir = new Path(snapshotDir, regionName); - HRegionInfo snapshotRegionInfo = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir); - assertEquals(info, snapshotRegionInfo); - // check to make sure we have the family - Path familyDir = new Path(regionDir, Bytes.toString(TEST_FAM)); - assertTrue("Missing region " + Bytes.toString(snapshotRegionInfo.getStartKey()), - fs.exists(familyDir)); - - // make sure we have some file references - assertTrue(fs.listStatus(familyDir).length > 0); - } + SnapshotTestingUtils.createSnapshotAndValidate(admin, + TableName.valueOf(STRING_TABLE_NAME), Bytes.toString(TEST_FAM), + snapshotName, rootDir, fs, true); } /**