hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mberto...@apache.org
Subject svn commit: r1593139 [5/6] - in /hbase/trunk: hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ hbase-protocol/src/main/protobuf/ hbase-server/src/main/java/org/apache/hadoop/hbase/client/ hbase-server/src/main/java/org/apache/ha...
Date Wed, 07 May 2014 21:28:13 GMT
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java?rev=1593139&r1=1593138&r2=1593139&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java Wed May  7 21:28:12 2014
@@ -19,24 +19,28 @@
 package org.apache.hadoop.hbase.snapshot;
 
 import java.io.IOException;
-import java.io.FileNotFoundException;
+import java.io.InterruptedIOException;
 import java.util.HashSet;
-import java.util.LinkedList;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
-import java.util.TreeMap;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorCompletionService;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.io.HFileLink;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
 import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
-import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
 import org.apache.hadoop.hbase.util.FSVisitor;
 
 /**
@@ -44,8 +48,15 @@ import org.apache.hadoop.hbase.util.FSVi
  */
 @InterfaceAudience.Private
 public final class SnapshotReferenceUtil {
-  public interface FileVisitor extends FSVisitor.StoreFileVisitor,
-    FSVisitor.RecoveredEditsVisitor, FSVisitor.LogFileVisitor {
+  public static final Log LOG = LogFactory.getLog(SnapshotReferenceUtil.class);
+
+  public interface StoreFileVisitor {
+    void storeFile(final HRegionInfo regionInfo, final String familyName,
+       final SnapshotRegionManifest.StoreFile storeFile) throws IOException;
+  }
+
+  public interface SnapshotVisitor extends StoreFileVisitor,
+    FSVisitor.LogFileVisitor {
   }
 
   private SnapshotReferenceUtil() {
@@ -64,80 +75,79 @@ public final class SnapshotReferenceUtil
   }
 
   /**
-   * Get the snapshotted recovered.edits dir for the specified region.
-   *
-   * @param snapshotDir directory where the specific snapshot is stored
-   * @param regionName name of the region
-   * @return path to the recovered.edits directory for the specified region files.
-   */
-  public static Path getRecoveredEditsDir(Path snapshotDir, String regionName) {
-    return HLogUtil.getRegionDirRecoveredEditsDir(new Path(snapshotDir, regionName));
-  }
-
-  /**
-   * Get the snapshot recovered.edits file
+   * Iterate over the snapshot store files, restored.edits and logs
    *
-   * @param snapshotDir directory where the specific snapshot is stored
-   * @param regionName name of the region
-   * @param logfile name of the edit file
-   * @return full path of the log file for the specified region files.
+   * @param conf The current {@link Configuration} instance.
+   * @param fs {@link FileSystem}
+   * @param snapshotDir {@link Path} to the Snapshot directory
+   * @param visitor callback object to get the referenced files
+   * @throws IOException if an error occurred while scanning the directory
    */
-  public static Path getRecoveredEdits(Path snapshotDir, String regionName, String logfile) {
-    return new Path(getRecoveredEditsDir(snapshotDir, regionName), logfile);
+  public static void visitReferencedFiles(final Configuration conf, final FileSystem fs,
+      final Path snapshotDir, final SnapshotVisitor visitor)
+      throws IOException {
+    SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
+    visitReferencedFiles(conf, fs, snapshotDir, desc, visitor);
   }
 
   /**
    * Iterate over the snapshot store files, restored.edits and logs
    *
+   * @param conf The current {@link Configuration} instance.
    * @param fs {@link FileSystem}
    * @param snapshotDir {@link Path} to the Snapshot directory
+   * @param desc the {@link SnapshotDescription} of the snapshot to verify
    * @param visitor callback object to get the referenced files
    * @throws IOException if an error occurred while scanning the directory
    */
-  public static void visitReferencedFiles(final FileSystem fs, final Path snapshotDir,
-      final FileVisitor visitor) throws IOException {
-    visitTableStoreFiles(fs, snapshotDir, visitor);
-    visitRecoveredEdits(fs, snapshotDir, visitor);
+  public static void visitReferencedFiles(final Configuration conf, final FileSystem fs,
+      final Path snapshotDir, final SnapshotDescription desc, final SnapshotVisitor visitor)
+      throws IOException {
+    visitTableStoreFiles(conf, fs, snapshotDir, desc, visitor);
     visitLogFiles(fs, snapshotDir, visitor);
   }
 
   /**
    * Iterate over the snapshot store files
    *
+   * @param conf The current {@link Configuration} instance.
    * @param fs {@link FileSystem}
    * @param snapshotDir {@link Path} to the Snapshot directory
+   * @param desc the {@link SnapshotDescription} of the snapshot to verify
    * @param visitor callback object to get the store files
    * @throws IOException if an error occurred while scanning the directory
    */
-  public static void visitTableStoreFiles(final FileSystem fs, final Path snapshotDir,
-      final FSVisitor.StoreFileVisitor visitor) throws IOException {
-    FSVisitor.visitTableStoreFiles(fs, snapshotDir, visitor);
+  public static void visitTableStoreFiles(final Configuration conf, final FileSystem fs,
+      final Path snapshotDir, final SnapshotDescription desc, final StoreFileVisitor visitor)
+      throws IOException {
+    SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, desc);
+    List<SnapshotRegionManifest> regionManifests = manifest.getRegionManifests();
+    if (regionManifests == null || regionManifests.size() == 0) {
+      LOG.debug("No manifest files present: " + snapshotDir);
+      return;
+    }
+
+    for (SnapshotRegionManifest regionManifest: regionManifests) {
+      visitRegionStoreFiles(regionManifest, visitor);
+    }
   }
 
   /**
    * Iterate over the snapshot store files in the specified region
    *
-   * @param fs {@link FileSystem}
-   * @param regionDir {@link Path} to the Snapshot region directory
+   * @param manifest snapshot manifest to inspect
    * @param visitor callback object to get the store files
    * @throws IOException if an error occurred while scanning the directory
    */
-  public static void visitRegionStoreFiles(final FileSystem fs, final Path regionDir,
-      final FSVisitor.StoreFileVisitor visitor) throws IOException {
-    FSVisitor.visitRegionStoreFiles(fs, regionDir, visitor);
-  }
-
-  /**
-   * Iterate over the snapshot recovered.edits
-   *
-   * @param fs {@link FileSystem}
-   * @param snapshotDir {@link Path} to the Snapshot directory
-   * @param visitor callback object to get the recovered.edits files
-   * @throws IOException if an error occurred while scanning the directory
-   */
-  public static void visitRecoveredEdits(final FileSystem fs, final Path snapshotDir,
-      final FSVisitor.RecoveredEditsVisitor visitor) throws IOException {
-    FSVisitor.visitTableRecoveredEdits(fs, snapshotDir, visitor);
+  public static void visitRegionStoreFiles(final SnapshotRegionManifest manifest,
+      final StoreFileVisitor visitor) throws IOException {
+    HRegionInfo regionInfo = HRegionInfo.convert(manifest.getRegionInfo());
+    for (SnapshotRegionManifest.FamilyFiles familyFiles: manifest.getFamilyFilesList()) {
+      String familyName = familyFiles.getFamilyName().toStringUtf8();
+      for (SnapshotRegionManifest.StoreFile storeFile: familyFiles.getStoreFilesList()) {
+        visitor.storeFile(regionInfo, familyName, storeFile);
+      }
+    }
   }
 
   /**
@@ -165,85 +175,151 @@ public final class SnapshotReferenceUtil
    */
   public static void verifySnapshot(final Configuration conf, final FileSystem fs,
       final Path snapshotDir, final SnapshotDescription snapshotDesc) throws IOException {
-    final TableName table = TableName.valueOf(snapshotDesc.getTable());
-    visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() {
-      public void storeFile (final String region, final String family, final String hfile)
-          throws IOException {
-        HFileLink link = HFileLink.create(conf, table, region, family, hfile);
-        try {
-          link.getFileStatus(fs);
-        } catch (FileNotFoundException e) {
-          throw new CorruptedSnapshotException("Corrupted snapshot '" + snapshotDesc + "'", e);
+    SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
+    verifySnapshot(conf, fs, manifest);
+  }
+
+  /**
+   * Verify the validity of the snapshot
+   *
+   * @param conf The current {@link Configuration} instance.
+   * @param fs {@link FileSystem}
+   * @param manifest snapshot manifest to inspect
+   * @throws CorruptedSnapshotException if the snapshot is corrupted
+   * @throws IOException if an error occurred while scanning the directory
+   */
+  public static void verifySnapshot(final Configuration conf, final FileSystem fs,
+      final SnapshotManifest manifest) throws IOException {
+    final SnapshotDescription snapshotDesc = manifest.getSnapshotDescription();
+    final Path snapshotDir = manifest.getSnapshotDir();
+
+    List<SnapshotRegionManifest> regionManifests = manifest.getRegionManifests();
+    if (regionManifests == null || regionManifests.size() == 0) {
+      LOG.debug("No manifest files present: " + snapshotDir);
+      return;
+    }
+
+    ExecutorService exec = SnapshotManifest.createExecutor(conf, "VerifySnapshot");
+    final ExecutorCompletionService<Void> completionService =
+      new ExecutorCompletionService<Void>(exec);
+    try {
+      for (final SnapshotRegionManifest regionManifest: regionManifests) {
+        completionService.submit(new Callable<Void>() {
+          @Override
+          public Void call() throws IOException {
+            visitRegionStoreFiles(regionManifest, new StoreFileVisitor() {
+              @Override
+              public void storeFile(final HRegionInfo regionInfo, final String family,
+                  final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+                verifyStoreFile(conf, fs, snapshotDir, snapshotDesc, regionInfo, family, storeFile);
+              }
+            });
+            return null;
+          }
+        });
+      }
+      try {
+        for (int i = 0; i < regionManifests.size(); ++i) {
+          completionService.take().get();
+        }
+      } catch (InterruptedException e) {
+        throw new InterruptedIOException(e.getMessage());
+      } catch (ExecutionException e) {
+        if (e.getCause() instanceof CorruptedSnapshotException) {
+          throw new CorruptedSnapshotException(e.getCause().getMessage(), snapshotDesc);
+        } else {
+          IOException ex = new IOException();
+          ex.initCause(e.getCause());
+          throw ex;
         }
       }
-    });
+    } finally {
+      exec.shutdown();
+    }
   }
 
   /**
-   * Returns the set of region names available in the snapshot.
+   * Verify the validity of the snapshot store file
    *
+   * @param conf The current {@link Configuration} instance.
    * @param fs {@link FileSystem}
-   * @param snapshotDir {@link Path} to the Snapshot directory
+   * @param snapshotDir {@link Path} to the Snapshot directory of the snapshot to verify
+   * @param snapshot the {@link SnapshotDescription} of the snapshot to verify
+   * @param regionInfo {@link HRegionInfo} of the region that contains the store file
+   * @param family family that contains the store file
+   * @param storeFile the store file to verify
+   * @throws CorruptedSnapshotException if the snapshot is corrupted
    * @throws IOException if an error occurred while scanning the directory
-   * @return the set of the regions contained in the snapshot
    */
-  public static Set<String> getSnapshotRegionNames(final FileSystem fs, final Path snapshotDir)
-      throws IOException {
-    FileStatus[] regionDirs = FSUtils.listStatus(fs, snapshotDir, new FSUtils.RegionDirFilter(fs));
-    if (regionDirs == null) return null;
+  public static void verifyStoreFile(final Configuration conf, final FileSystem fs,
+      final Path snapshotDir, final SnapshotDescription snapshot, final HRegionInfo regionInfo,
+      final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+    String fileName = storeFile.getName();
+
+    Path refPath = null;
+    if (StoreFileInfo.isReference(fileName)) {
+      // If is a reference file check if the parent file is present in the snapshot
+      Path snapshotHFilePath = new Path(new Path(
+          new Path(snapshotDir, regionInfo.getEncodedName()), family), fileName);
+      refPath = StoreFileInfo.getReferredToFile(snapshotHFilePath);
+      if (!fs.exists(refPath)) {
+        throw new CorruptedSnapshotException("Missing parent hfile for: " + fileName, snapshot);
+      }
+    }
+
+    Path linkPath;
+    if (refPath != null && HFileLink.isHFileLink(refPath)) {
+      linkPath = new Path(family, refPath.getName());
+    } else if (HFileLink.isHFileLink(fileName)) {
+      linkPath = new Path(family, fileName);
+    } else {
+      linkPath = new Path(family, HFileLink.createHFileLinkName(
+        TableName.valueOf(snapshot.getTable()), regionInfo.getEncodedName(), fileName));
+    }
 
-    Set<String> regions = new HashSet<String>();
-    for (FileStatus regionDir: regionDirs) {
-      regions.add(regionDir.getPath().getName());
+    // check if the linked file exists (in the archive, or in the table dir)
+    HFileLink link = new HFileLink(conf, linkPath);
+    if (!link.exists(fs)) {
+      throw new CorruptedSnapshotException("Can't find hfile: " + fileName
+          + " in the real (" + link.getOriginPath() + ") or archive (" + link.getArchivePath()
+          + ") directory for the primary table.", snapshot);
     }
-    return regions;
   }
 
   /**
-   * Get the list of hfiles for the specified snapshot region.
-   * NOTE: The current implementation keeps one empty file per HFile in the region.
-   * The file name matches the one in the original table, and by reconstructing
-   * the path you can quickly jump to the referenced file.
+   * Returns the store file names in the snapshot.
    *
+   * @param conf The current {@link Configuration} instance.
    * @param fs {@link FileSystem}
-   * @param snapshotRegionDir {@link Path} to the Snapshot region directory
-   * @return Map of hfiles per family, the key is the family name and values are hfile names
+   * @param snapshotDir {@link Path} to the Snapshot directory
    * @throws IOException if an error occurred while scanning the directory
+   * @return the names of hfiles in the specified snaphot
    */
-  public static Map<String, List<String>> getRegionHFileReferences(final FileSystem fs,
-      final Path snapshotRegionDir) throws IOException {
-    final Map<String, List<String>> familyFiles = new TreeMap<String, List<String>>();
-
-    visitRegionStoreFiles(fs, snapshotRegionDir,
-      new FSVisitor.StoreFileVisitor() {
-        public void storeFile (final String region, final String family, final String hfile)
-            throws IOException {
-          List<String> hfiles = familyFiles.get(family);
-          if (hfiles == null) {
-            hfiles = new LinkedList<String>();
-            familyFiles.put(family, hfiles);
-          }
-          hfiles.add(hfile);
-        }
-    });
-
-    return familyFiles;
+  public static Set<String> getHFileNames(final Configuration conf, final FileSystem fs,
+      final Path snapshotDir) throws IOException {
+    SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
+    return getHFileNames(conf, fs, snapshotDir, desc);
   }
 
   /**
    * Returns the store file names in the snapshot.
    *
+   * @param conf The current {@link Configuration} instance.
    * @param fs {@link FileSystem}
    * @param snapshotDir {@link Path} to the Snapshot directory
+   * @param snapshotDesc the {@link SnapshotDescription} of the snapshot to inspect
    * @throws IOException if an error occurred while scanning the directory
    * @return the names of hfiles in the specified snaphot
    */
-  public static Set<String> getHFileNames(final FileSystem fs, final Path snapshotDir)
+  private static Set<String> getHFileNames(final Configuration conf, final FileSystem fs,
+      final Path snapshotDir, final SnapshotDescription snapshotDesc)
       throws IOException {
     final Set<String> names = new HashSet<String>();
-    visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() {
-      public void storeFile (final String region, final String family, final String hfile)
-          throws IOException {
+    visitTableStoreFiles(conf, fs, snapshotDir, snapshotDesc, new StoreFileVisitor() {
+      @Override
+      public void storeFile(final HRegionInfo regionInfo, final String family,
+            final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+        String hfile = storeFile.getName();
         if (HFileLink.isHFileLink(hfile)) {
           names.add(HFileLink.getReferencedHFileName(hfile));
         } else {
@@ -266,6 +342,7 @@ public final class SnapshotReferenceUtil
       throws IOException {
     final Set<String> names = new HashSet<String>();
     visitLogFiles(fs, snapshotDir, new FSVisitor.LogFileVisitor() {
+      @Override
       public void logFile (final String server, final String logfile) throws IOException {
         names.add(logfile);
       }

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSVisitor.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSVisitor.java?rev=1593139&r1=1593138&r2=1593139&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSVisitor.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSVisitor.java Wed May  7 21:28:12 2014
@@ -150,7 +150,7 @@ public final class FSVisitor {
       final FSVisitor.RecoveredEditsVisitor visitor) throws IOException {
     FileStatus[] regions = FSUtils.listStatus(fs, tableDir, new FSUtils.RegionDirFilter(fs));
     if (regions == null) {
-      LOG.info("No regions under directory:" + tableDir);
+      LOG.info("No recoveredEdits regions under directory:" + tableDir);
       return;
     }
 

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java?rev=1593139&r1=1593138&r2=1593139&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java Wed May  7 21:28:12 2014
@@ -20,9 +20,7 @@ package org.apache.hadoop.hbase.client;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
 
-import java.util.HashSet;
 import java.util.List;
-import java.util.Set;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -39,9 +37,9 @@ import org.apache.hadoop.hbase.protobuf.
 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
 import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
 import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
+import org.apache.hadoop.hbase.snapshot.SnapshotManifestV1;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -182,15 +180,6 @@ public class TestSnapshotFromClient {
     HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
     UTIL.loadTable(table, TEST_FAM, false);
 
-    // get the name of all the regionservers hosting the snapshotted table
-    Set<String> snapshotServers = new HashSet<String>();
-    List<RegionServerThread> servers = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads();
-    for (RegionServerThread server : servers) {
-      if (server.getRegionServer().getOnlineRegions(TABLE_NAME).size() > 0) {
-        snapshotServers.add(server.getRegionServer().getServerName().toString());
-      }
-    }
-
     LOG.debug("FS state before disable:");
     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
@@ -203,8 +192,16 @@ public class TestSnapshotFromClient {
       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
 
     // take a snapshot of the disabled table
-    byte[] snapshot = Bytes.toBytes("offlineTableSnapshot");
-    admin.snapshot(snapshot, TABLE_NAME);
+    final String SNAPSHOT_NAME = "offlineTableSnapshot";
+    byte[] snapshot = Bytes.toBytes(SNAPSHOT_NAME);
+
+    SnapshotDescription desc = SnapshotDescription.newBuilder()
+      .setType(SnapshotDescription.Type.DISABLED)
+      .setTable(STRING_TABLE_NAME)
+      .setName(SNAPSHOT_NAME)
+      .setVersion(SnapshotManifestV1.DESCRIPTOR_VERSION)
+      .build();
+    admin.snapshot(desc);
     LOG.debug("Snapshot completed.");
 
     // make sure we have the snapshot
@@ -219,7 +216,7 @@ public class TestSnapshotFromClient {
       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
 
     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
-      admin, fs, false, new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), snapshotServers);
+      admin, fs);
 
     admin.deleteSnapshot(snapshot);
     snapshots = admin.listSnapshots();
@@ -263,15 +260,6 @@ public class TestSnapshotFromClient {
     // make sure we don't fail on listing snapshots
     SnapshotTestingUtils.assertNoSnapshots(admin);
 
-    // get the name of all the regionservers hosting the snapshotted table
-    Set<String> snapshotServers = new HashSet<String>();
-    List<RegionServerThread> servers = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads();
-    for (RegionServerThread server : servers) {
-      if (server.getRegionServer().getOnlineRegions(TABLE_NAME).size() > 0) {
-        snapshotServers.add(server.getRegionServer().getServerName().toString());
-      }
-    }
-
     LOG.debug("FS state before disable:");
     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
@@ -299,8 +287,8 @@ public class TestSnapshotFromClient {
 
     List<byte[]> emptyCfs = Lists.newArrayList(TEST_FAM); // no file in the region
     List<byte[]> nonEmptyCfs = Lists.newArrayList();
-    SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, nonEmptyCfs, emptyCfs, rootDir,
-      admin, fs, false, new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), snapshotServers);
+    SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, nonEmptyCfs, emptyCfs,
+      rootDir, admin, fs);
 
     admin.deleteSnapshot(snapshot);
     snapshots = admin.listSnapshots();

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java?rev=1593139&r1=1593138&r2=1593139&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java Wed May  7 21:28:12 2014
@@ -29,13 +29,13 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellScanner;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.LargeTests;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.mapreduce.TestTableSnapshotInputFormat;
 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
 import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Test;
@@ -94,7 +94,7 @@ public class TestTableSnapshotScanner {
     HTable table = new HTable(util.getConfiguration(), tableName);
     util.loadTable(table, FAMILIES);
 
-    Path rootDir = new Path(util.getConfiguration().get(HConstants.HBASE_DIR));
+    Path rootDir = FSUtils.getRootDir(util.getConfiguration());
     FileSystem fs = rootDir.getFileSystem(util.getConfiguration());
 
     SnapshotTestingUtils.createSnapshotAndValidate(admin, tableName,
@@ -124,7 +124,7 @@ public class TestTableSnapshotScanner {
     testScanner(UTIL, "testWithMultiRegion", 20, true);
   }
 
-  private void testScanner(HBaseTestingUtility util, String snapshotName, int numRegions, 
+  private void testScanner(HBaseTestingUtility util, String snapshotName, int numRegions,
       boolean shutdownCluster) throws Exception {
     setupCluster();
     TableName tableName = TableName.valueOf("testScanner");
@@ -138,7 +138,7 @@ public class TestTableSnapshotScanner {
       Path restoreDir = util.getDataTestDirOnTestFS(snapshotName);
       Scan scan = new Scan(bbb, yyy); // limit the scan
 
-      TableSnapshotScanner scanner = new TableSnapshotScanner(UTIL.getConfiguration(), restoreDir, 
+      TableSnapshotScanner scanner = new TableSnapshotScanner(UTIL.getConfiguration(), restoreDir,
         snapshotName, scan);
 
       verifyScanner(scanner, bbb, yyy);
@@ -155,7 +155,7 @@ public class TestTableSnapshotScanner {
   private void verifyScanner(ResultScanner scanner, byte[] startRow, byte[] stopRow)
       throws IOException, InterruptedException {
 
-    HBaseTestingUtility.SeenRowTracker rowTracker = 
+    HBaseTestingUtility.SeenRowTracker rowTracker =
         new HBaseTestingUtility.SeenRowTracker(startRow, stopRow);
 
     while (true) {

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java?rev=1593139&r1=1593138&r2=1593139&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java Wed May  7 21:28:12 2014
@@ -32,7 +32,6 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellScanner;
-import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
@@ -44,12 +43,12 @@ import org.apache.hadoop.hbase.client.Re
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.io.hfile.BlockCache;
-import org.apache.hadoop.hbase.io.hfile.CacheConfig;
 import org.apache.hadoop.hbase.io.hfile.LruBlockCache;
 import org.apache.hadoop.hbase.mapreduce.TableSnapshotInputFormat.TableSnapshotRegionSplit;
 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
 import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.io.NullWritable;
 import org.apache.hadoop.mapreduce.InputSplit;
 import org.apache.hadoop.mapreduce.Job;
@@ -59,7 +58,6 @@ import org.apache.hadoop.mapreduce.TaskA
 import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
 import org.junit.After;
 import org.junit.Assert;
-import org.junit.Ignore;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
@@ -125,17 +123,17 @@ public class TestTableSnapshotInputForma
     Assert.assertEquals(Lists.newArrayList("h1"), tsif.getBestLocations(conf, blockDistribution));
 
     blockDistribution.addHostsAndBlockWeight(new String[] {"h2"}, 2);
-    Assert.assertEquals(Lists.newArrayList("h1", "h2"), 
+    Assert.assertEquals(Lists.newArrayList("h1", "h2"),
       tsif.getBestLocations(conf, blockDistribution));
 
     blockDistribution.addHostsAndBlockWeight(new String[] {"h2"}, 3);
-    Assert.assertEquals(Lists.newArrayList("h2", "h1"), 
+    Assert.assertEquals(Lists.newArrayList("h2", "h1"),
       tsif.getBestLocations(conf, blockDistribution));
 
     blockDistribution.addHostsAndBlockWeight(new String[] {"h3"}, 6);
     blockDistribution.addHostsAndBlockWeight(new String[] {"h4"}, 9);
 
-    Assert.assertEquals(Lists.newArrayList("h2", "h3", "h4", "h1"), 
+    Assert.assertEquals(Lists.newArrayList("h2", "h3", "h4", "h1"),
       tsif.getBestLocations(conf, blockDistribution));
   }
 
@@ -156,7 +154,7 @@ public class TestTableSnapshotInputForma
 
   public static class TestTableSnapshotReducer
     extends Reducer<ImmutableBytesWritable, NullWritable, NullWritable, NullWritable> {
-    HBaseTestingUtility.SeenRowTracker rowTracker = 
+    HBaseTestingUtility.SeenRowTracker rowTracker =
         new HBaseTestingUtility.SeenRowTracker(bbb, yyy);
     @Override
     protected void reduce(ImmutableBytesWritable key, Iterable<NullWritable> values,
@@ -191,7 +189,7 @@ public class TestTableSnapshotInputForma
     HTable table = new HTable(util.getConfiguration(), tableName);
     util.loadTable(table, FAMILIES);
 
-    Path rootDir = new Path(util.getConfiguration().get(HConstants.HBASE_DIR));
+    Path rootDir = FSUtils.getRootDir(util.getConfiguration());
     FileSystem fs = rootDir.getFileSystem(util.getConfiguration());
 
     SnapshotTestingUtils.createSnapshotAndValidate(admin, tableName,
@@ -250,7 +248,7 @@ public class TestTableSnapshotInputForma
     testWithMockedMapReduce(UTIL, "testWithMockedMapReduceMultiRegion", 10, 8);
   }
 
-  public void testWithMockedMapReduce(HBaseTestingUtility util, String snapshotName, 
+  public void testWithMockedMapReduce(HBaseTestingUtility util, String snapshotName,
       int numRegions, int expectedNumSplits) throws Exception {
     setupCluster();
     TableName tableName = TableName.valueOf("testWithMockedMapReduce");
@@ -282,7 +280,7 @@ public class TestTableSnapshotInputForma
 
     Assert.assertEquals(expectedNumSplits, splits.size());
 
-    HBaseTestingUtility.SeenRowTracker rowTracker = 
+    HBaseTestingUtility.SeenRowTracker rowTracker =
         new HBaseTestingUtility.SeenRowTracker(startRow, stopRow);
 
     for (int i = 0; i < splits.size(); i++) {
@@ -293,7 +291,7 @@ public class TestTableSnapshotInputForma
       // validate record reader
       TaskAttemptContext taskAttemptContext = mock(TaskAttemptContext.class);
       when(taskAttemptContext.getConfiguration()).thenReturn(job.getConfiguration());
-      RecordReader<ImmutableBytesWritable, Result> rr = 
+      RecordReader<ImmutableBytesWritable, Result> rr =
           tsif.createRecordReader(split, taskAttemptContext);
       rr.initialize(split, taskAttemptContext);
 
@@ -311,7 +309,7 @@ public class TestTableSnapshotInputForma
     rowTracker.validate();
   }
 
-  public static void verifyRowFromMap(ImmutableBytesWritable key, Result result) 
+  public static void verifyRowFromMap(ImmutableBytesWritable key, Result result)
       throws IOException {
     byte[] row = key.get();
     CellScanner scanner = result.cellScanner();
@@ -363,7 +361,7 @@ public class TestTableSnapshotInputForma
 
   // this is also called by the IntegrationTestTableSnapshotInputFormat
   public static void doTestWithMapReduce(HBaseTestingUtility util, TableName tableName,
-      String snapshotName, Path tableDir, int numRegions, int expectedNumSplits, 
+      String snapshotName, Path tableDir, int numRegions, int expectedNumSplits,
       boolean shutdownCluster) throws Exception {
 
     //create the table and snapshot
@@ -379,7 +377,7 @@ public class TestTableSnapshotInputForma
       Scan scan = new Scan(bbb, yyy); // limit the scan
 
       job.setJarByClass(util.getClass());
-      TableMapReduceUtil.addDependencyJars(job.getConfiguration(), 
+      TableMapReduceUtil.addDependencyJars(job.getConfiguration(),
         TestTableSnapshotInputFormat.class);
 
       TableMapReduceUtil.initTableSnapshotMapperJob(snapshotName,

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java?rev=1593139&r1=1593138&r2=1593139&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java Wed May  7 21:28:12 2014
@@ -26,6 +26,7 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.Set;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -52,6 +53,7 @@ import org.apache.hadoop.hbase.protobuf.
 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
 import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
 import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -330,19 +332,20 @@ public class TestSnapshotFromMaster {
 
     // get the snapshot files for the table
     Path snapshotTable = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
-    Path[] snapshotHFiles = SnapshotTestingUtils.listHFiles(fs, snapshotTable);
+    Set<String> snapshotHFiles = SnapshotReferenceUtil.getHFileNames(
+        UTIL.getConfiguration(), fs, snapshotTable);
     // check that the files in the archive contain the ones that we need for the snapshot
     LOG.debug("Have snapshot hfiles:");
-    for (Path file : snapshotHFiles) {
-      LOG.debug(file);
+    for (String fileName : snapshotHFiles) {
+      LOG.debug(fileName);
     }
     // get the archived files for the table
     Collection<String> files = getArchivedHFiles(archiveDir, rootDir, fs, TABLE_NAME);
 
     // and make sure that there is a proper subset
-    for (Path file : snapshotHFiles) {
-      assertTrue("Archived hfiles " + files + " is missing snapshot file:" + file,
-        files.contains(file.getName()));
+    for (String fileName : snapshotHFiles) {
+      assertTrue("Archived hfiles " + files + " is missing snapshot file:" + fileName,
+        files.contains(fileName));
     }
 
     // delete the existing snapshot

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java?rev=1593139&r1=1593138&r2=1593139&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java Wed May  7 21:28:12 2014
@@ -22,7 +22,9 @@ import static org.junit.Assert.assertTru
 
 import java.io.IOException;
 import java.util.Collection;
+import java.util.ArrayList;
 import java.util.HashSet;
+import java.util.List;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -31,9 +33,10 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.MediumTests;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
 import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
 import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
-import org.apache.hadoop.hbase.snapshot.TakeSnapshotUtils;
+import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils.SnapshotMock;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.junit.After;
 import org.junit.AfterClass;
@@ -49,6 +52,7 @@ public class TestSnapshotFileCache {
 
   private static final Log LOG = LogFactory.getLog(TestSnapshotFileCache.class);
   private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+  private static long sequenceId = 0;
   private static FileSystem fs;
   private static Path rootDir;
 
@@ -72,86 +76,17 @@ public class TestSnapshotFileCache {
   }
 
   @Test(timeout = 10000000)
-  public void testLoadAndDelete() throws Exception {
+  public void testLoadAndDelete() throws IOException {
     // don't refresh the cache unless we tell it to
     long period = Long.MAX_VALUE;
-    Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
     SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
         "test-snapshot-file-cache-refresh", new SnapshotFiles());
 
-    Path snapshot = new Path(snapshotDir, "snapshot");
-    Path region = new Path(snapshot, "7e91021");
-    Path family = new Path(region, "fam");
-    Path file1 = new Path(family, "file1");
-    Path file2 = new Path(family, "file2");
-
-    // create two hfiles under the snapshot
-    fs.createNewFile(file1);
-    fs.createNewFile(file2);
+    createAndTestSnapshotV1(cache, "snapshot1a", false, true);
+    createAndTestSnapshotV1(cache, "snapshot1b", true, true);
 
-    FSUtils.logFileSystemState(fs, rootDir, LOG);
-
-    // then make sure the cache finds them
-    assertTrue("Cache didn't find:" + file1, cache.contains(file1.getName()));
-    assertTrue("Cache didn't find:" + file2, cache.contains(file2.getName()));
-    String not = "file-shouldn't-be-found";
-    assertFalse("Cache found '" + not + "', but it shouldn't have.", cache.contains(not));
-
-    // make sure we get a little bit of separation in the modification times
-    // its okay if we sleep a little longer (b/c of GC pause), as long as we sleep a little
-    Thread.sleep(10);
-
-    LOG.debug("Deleting snapshot.");
-    // then delete the snapshot and make sure that we can still find the files
-    if (!fs.delete(snapshot, true)) {
-      throw new IOException("Couldn't delete " + snapshot + " for an unknown reason.");
-    }
-    FSUtils.logFileSystemState(fs, rootDir, LOG);
-
-
-    LOG.debug("Checking to see if file is deleted.");
-    assertTrue("Cache didn't find:" + file1, cache.contains(file1.getName()));
-    assertTrue("Cache didn't find:" + file2, cache.contains(file2.getName()));
-
-    // then trigger a refresh
-    cache.triggerCacheRefreshForTesting();
-    // and not it shouldn't find those files
-    assertFalse("Cache found '" + file1 + "', but it shouldn't have.",
-      cache.contains(file1.getName()));
-    assertFalse("Cache found '" + file2 + "', but it shouldn't have.",
-      cache.contains(file2.getName()));
-
-    fs.delete(snapshotDir, true);
-  }
-
-  @Test
-  public void testLoadsTmpDir() throws Exception {
-    // don't refresh the cache unless we tell it to
-    long period = Long.MAX_VALUE;
-    Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
-    SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
-        "test-snapshot-file-cache-refresh", new SnapshotFiles());
-
-    // create a file in a 'completed' snapshot
-    Path snapshot = new Path(snapshotDir, "snapshot");
-    Path region = new Path(snapshot, "7e91021");
-    Path family = new Path(region, "fam");
-    Path file1 = new Path(family, "file1");
-    fs.createNewFile(file1);
-
-    // create an 'in progress' snapshot
-    SnapshotDescription desc = SnapshotDescription.newBuilder().setName("working").build();
-    snapshot = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir);
-    region = new Path(snapshot, "7e91021");
-    family = new Path(region, "fam");
-    Path file2 = new Path(family, "file2");
-    fs.createNewFile(file2);
-
-    FSUtils.logFileSystemState(fs, rootDir, LOG);
-
-    // then make sure the cache finds both files
-    assertTrue("Cache didn't find:" + file1, cache.contains(file1.getName()));
-    assertTrue("Cache didn't find:" + file2, cache.contains(file2.getName()));
+    createAndTestSnapshotV2(cache, "snapshot2a", false, true);
+    createAndTestSnapshotV2(cache, "snapshot2b", true, true);
   }
 
   @Test
@@ -168,14 +103,14 @@ public class TestSnapshotFileCache {
         });
 
     // create a file in a 'completed' snapshot
-    Path snapshot = new Path(snapshotDir, "snapshot");
-    Path region = new Path(snapshot, "7e91021");
-    Path family = new Path(region, "fam");
-    Path file1 = new Path(family, "file1");
+    SnapshotDescription desc = SnapshotDescription.newBuilder().setName("snapshot").build();
+    Path snapshot = SnapshotDescriptionUtils.getCompletedSnapshotDir(desc, rootDir);
+    SnapshotDescriptionUtils.writeSnapshotInfo(desc, snapshot, fs);
+    Path file1 = new Path(new Path(new Path(snapshot, "7e91021"), "fam"), "file1");
     fs.createNewFile(file1);
 
     // and another file in the logs directory
-    Path logs = TakeSnapshotUtils.getSnapshotHLogsDir(snapshot, "server");
+    Path logs = getSnapshotHLogsDir(snapshot, "server");
     Path log = new Path(logs, "me.hbase.com%2C58939%2C1350424310315.1350424315552");
     fs.createNewFile(log);
 
@@ -187,65 +122,117 @@ public class TestSnapshotFileCache {
     assertTrue("Cache didn't find:" + log, cache.contains(log.getName()));
   }
 
+  /**
+   * Get the log directory for a specific snapshot
+   * @param snapshotDir directory where the specific snapshot will be store
+   * @param serverName name of the parent regionserver for the log files
+   * @return path to the log home directory for the archive files.
+   */
+  public static Path getSnapshotHLogsDir(Path snapshotDir, String serverName) {
+    return new Path(snapshotDir, HLogUtil.getHLogDirectoryName(serverName));
+  }
+
   @Test
   public void testReloadModifiedDirectory() throws IOException {
     // don't refresh the cache unless we tell it to
     long period = Long.MAX_VALUE;
-    Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
     SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
         "test-snapshot-file-cache-refresh", new SnapshotFiles());
 
-    Path snapshot = new Path(snapshotDir, "snapshot");
-    Path region = new Path(snapshot, "7e91021");
-    Path family = new Path(region, "fam");
-    Path file1 = new Path(family, "file1");
-    Path file2 = new Path(family, "file2");
-
-    // create two hfiles under the snapshot
-    fs.createNewFile(file1);
-    fs.createNewFile(file2);
-
-    FSUtils.logFileSystemState(fs, rootDir, LOG);
-
-    assertTrue("Cache didn't find " + file1, cache.contains(file1.getName()));
-
+    createAndTestSnapshotV1(cache, "snapshot1", false, true);
     // now delete the snapshot and add a file with a different name
-    fs.delete(snapshot, true);
-    Path file3 = new Path(family, "new_file");
-    fs.createNewFile(file3);
+    createAndTestSnapshotV1(cache, "snapshot1", false, false);
 
-    FSUtils.logFileSystemState(fs, rootDir, LOG);
-    assertTrue("Cache didn't find new file:" + file3, cache.contains(file3.getName()));
+    createAndTestSnapshotV2(cache, "snapshot2", false, true);
+    // now delete the snapshot and add a file with a different name
+    createAndTestSnapshotV2(cache, "snapshot2", false, false);
   }
 
   @Test
   public void testSnapshotTempDirReload() throws IOException {
     long period = Long.MAX_VALUE;
     // This doesn't refresh cache until we invoke it explicitly
-    Path snapshotDir = new Path(SnapshotDescriptionUtils.getSnapshotsDir(rootDir),
-        SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME);
     SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
         "test-snapshot-file-cache-refresh", new SnapshotFiles());
 
-    // Add a new snapshot
-    Path snapshot1 = new Path(snapshotDir, "snapshot1");
-    Path file1 = new Path(new Path(new Path(snapshot1, "7e91021"), "fam"), "file1");
-    fs.createNewFile(file1);
-    assertTrue(cache.contains(file1.getName()));
+    // Add a new non-tmp snapshot
+    createAndTestSnapshotV1(cache, "snapshot0v1", false, false);
+    createAndTestSnapshotV1(cache, "snapshot0v2", false, false);
+
+    // Add a new tmp snapshot
+    createAndTestSnapshotV2(cache, "snapshot1", true, false);
 
-    // Add another snapshot
-    Path snapshot2 = new Path(snapshotDir, "snapshot2");
-    Path file2 = new Path(new Path(new Path(snapshot2, "7e91021"), "fam2"), "file2");
-    fs.createNewFile(file2);
-    assertTrue(cache.contains(file2.getName()));
+    // Add another tmp snapshot
+    createAndTestSnapshotV2(cache, "snapshot2", true, false);
   }
 
   class SnapshotFiles implements SnapshotFileCache.SnapshotFileInspector {
     public Collection<String> filesUnderSnapshot(final Path snapshotDir) throws IOException {
       Collection<String> files =  new HashSet<String>();
       files.addAll(SnapshotReferenceUtil.getHLogNames(fs, snapshotDir));
-      files.addAll(SnapshotReferenceUtil.getHFileNames(fs, snapshotDir));
+      files.addAll(SnapshotReferenceUtil.getHFileNames(UTIL.getConfiguration(), fs, snapshotDir));
       return files;
     }
   };
+
+  private void createAndTestSnapshotV1(final SnapshotFileCache cache, final String name,
+      final boolean tmp, final boolean removeOnExit) throws IOException {
+    SnapshotMock snapshotMock = new SnapshotMock(UTIL.getConfiguration(), fs, rootDir);
+    SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV1(name);
+    createAndTestSnapshot(cache, builder, tmp, removeOnExit);
+  }
+
+  private void createAndTestSnapshotV2(final SnapshotFileCache cache, final String name,
+      final boolean tmp, final boolean removeOnExit) throws IOException {
+    SnapshotMock snapshotMock = new SnapshotMock(UTIL.getConfiguration(), fs, rootDir);
+    SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2(name);
+    createAndTestSnapshot(cache, builder, tmp, removeOnExit);
+  }
+
+  private void createAndTestSnapshot(final SnapshotFileCache cache,
+      final SnapshotMock.SnapshotBuilder builder,
+      final boolean tmp, final boolean removeOnExit) throws IOException {
+    List<String> files = new ArrayList<String>();
+    for (int i = 0; i < 3; ++i) {
+      for (Path filePath: builder.addRegion()) {
+        String fileName = filePath.getName();
+        if (tmp) {
+          // We should be able to find all the files while the snapshot creation is in-progress
+          FSUtils.logFileSystemState(fs, rootDir, LOG);
+          assertTrue("Cache didn't find " + fileName, cache.contains(fileName));
+        }
+        files.add(fileName);
+      }
+    }
+
+    // Finalize the snapshot
+    if (!tmp) {
+      builder.commit();
+    }
+
+    // Make sure that all files are still present
+    for (String fileName: files) {
+      assertTrue("Cache didn't find " + fileName, cache.contains(fileName));
+    }
+
+    FSUtils.logFileSystemState(fs, rootDir, LOG);
+    if (removeOnExit) {
+      LOG.debug("Deleting snapshot.");
+      fs.delete(builder.getSnapshotsDir(), true);
+      FSUtils.logFileSystemState(fs, rootDir, LOG);
+
+      // The files should be in cache until next refresh
+      for (String fileName: files) {
+        assertTrue("Cache didn't find " + fileName, cache.contains(fileName));
+      }
+
+      // then trigger a refresh
+      cache.triggerCacheRefreshForTesting();
+      // and not it shouldn't find those files
+      for (String fileName: files) {
+        assertFalse("Cache found '" + fileName + "', but it shouldn't have.",
+                    cache.contains(fileName));
+      }
+    }
+  }
 }

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java?rev=1593139&r1=1593138&r2=1593139&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java Wed May  7 21:28:12 2014
@@ -22,14 +22,18 @@ import static org.junit.Assert.assertFal
 import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.TreeSet;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
@@ -43,11 +47,13 @@ import org.apache.hadoop.hbase.client.Du
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
 import org.apache.hadoop.hbase.io.HFileLink;
 import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.master.MasterFileSystem;
 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneRequest;
 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneResponse;
 import org.apache.hadoop.hbase.regionserver.HRegion;
@@ -56,6 +62,7 @@ import org.apache.hadoop.hbase.regionser
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSTableDescriptors;
 import org.apache.hadoop.hbase.util.FSVisitor;
+import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.MD5Hash;
 import org.junit.Assert;
 
@@ -144,14 +151,12 @@ public class SnapshotTestingUtils {
    */
   public static void confirmSnapshotValid(
       SnapshotDescription snapshotDescriptor, TableName tableName,
-      byte[] testFamily, Path rootDir, HBaseAdmin admin, FileSystem fs,
-      boolean requireLogs, Path logsDir, Set<String> snapshotServers)
+      byte[] testFamily, Path rootDir, HBaseAdmin admin, FileSystem fs)
       throws IOException {
     ArrayList nonEmptyTestFamilies = new ArrayList(1);
     nonEmptyTestFamilies.add(testFamily);
     confirmSnapshotValid(snapshotDescriptor, tableName,
-      nonEmptyTestFamilies, null, rootDir, admin, fs, requireLogs,
-      logsDir, snapshotServers);
+      nonEmptyTestFamilies, null, rootDir, admin, fs);
   }
 
   /**
@@ -159,14 +164,12 @@ public class SnapshotTestingUtils {
    */
   public static void confirmEmptySnapshotValid(
       SnapshotDescription snapshotDescriptor, TableName tableName,
-      byte[] testFamily, Path rootDir, HBaseAdmin admin, FileSystem fs,
-      boolean requireLogs, Path logsDir, Set<String> snapshotServers)
+      byte[] testFamily, Path rootDir, HBaseAdmin admin, FileSystem fs)
       throws IOException {
     ArrayList emptyTestFamilies = new ArrayList(1);
     emptyTestFamilies.add(testFamily);
     confirmSnapshotValid(snapshotDescriptor, tableName,
-      null, emptyTestFamilies, rootDir, admin, fs, requireLogs,
-      logsDir, snapshotServers);
+      null, emptyTestFamilies, rootDir, admin, fs);
   }
 
   /**
@@ -178,45 +181,31 @@ public class SnapshotTestingUtils {
   public static void confirmSnapshotValid(
       SnapshotDescription snapshotDescriptor, TableName tableName,
       List<byte[]> nonEmptyTestFamilies, List<byte[]> emptyTestFamilies,
-      Path rootDir, HBaseAdmin admin, FileSystem fs, boolean requireLogs,
-      Path logsDir, Set<String> snapshotServers) throws IOException {
+      Path rootDir, HBaseAdmin admin, FileSystem fs) throws IOException {
+    final Configuration conf = admin.getConfiguration();
+
     // check snapshot dir
     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
         snapshotDescriptor, rootDir);
     assertTrue(fs.exists(snapshotDir));
 
-    // check snapshot info
-    Path snapshotinfo = new Path(snapshotDir, SnapshotDescriptionUtils.SNAPSHOTINFO_FILE);
-    assertTrue(fs.exists(snapshotinfo));
-
-    // check the logs dir
-    if (requireLogs) {
-      TakeSnapshotUtils.verifyAllLogsGotReferenced(fs, logsDir,
-          snapshotServers, snapshotDescriptor, new Path(snapshotDir,
-              HConstants.HREGION_LOGDIR_NAME));
-    }
-
-    // check the table info
-    HTableDescriptor desc = FSTableDescriptors.getTableDescriptorFromFs(fs, rootDir, tableName);
-    HTableDescriptor snapshotDesc = FSTableDescriptors.getTableDescriptorFromFs(fs, snapshotDir);
-    assertEquals(desc, snapshotDesc);
+    SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
 
     // Extract regions and families with store files
-    final Set<String> snapshotRegions = new HashSet<String>();
     final Set<byte[]> snapshotFamilies = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
-    FSVisitor.visitRegions(fs, snapshotDir, new FSVisitor.RegionVisitor() {
-      @Override
-      public void region(String region) throws IOException {
-        snapshotRegions.add(region);
-      }
-    });
-    FSVisitor.visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() {
-      @Override
-      public void storeFile(final String region, final String family, final String hfileName)
-          throws IOException {
-        snapshotFamilies.add(Bytes.toBytes(family));
-      }
-    });
+
+    SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, desc);
+    Map<String, SnapshotRegionManifest> regionManifests = manifest.getRegionManifestsMap();
+    for (SnapshotRegionManifest regionManifest: regionManifests.values()) {
+      SnapshotReferenceUtil.visitRegionStoreFiles(regionManifest,
+          new SnapshotReferenceUtil.StoreFileVisitor() {
+        @Override
+        public void storeFile(final HRegionInfo regionInfo, final String family,
+              final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+          snapshotFamilies.add(Bytes.toBytes(family));
+        }
+      });
+    }
 
     // Verify that there are store files in the specified families
     if (nonEmptyTestFamilies != null) {
@@ -234,16 +223,12 @@ public class SnapshotTestingUtils {
 
     // check the region snapshot for all the regions
     List<HRegionInfo> regions = admin.getTableRegions(tableName);
-    assertEquals(regions.size(), snapshotRegions.size());
+    assertEquals(regions.size(), regionManifests.size());
 
-    // Verify Regions
+    // Verify Regions (redundant check, see MasterSnapshotVerifier)
     for (HRegionInfo info : regions) {
       String regionName = info.getEncodedName();
-      assertTrue(snapshotRegions.contains(regionName));
-
-      Path regionDir = new Path(snapshotDir, regionName);
-      HRegionInfo snapshotRegionInfo = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
-      assertEquals(info, snapshotRegionInfo);
+      assertTrue(regionManifests.containsKey(regionName));
     }
   }
 
@@ -351,6 +336,17 @@ public class SnapshotTestingUtils {
     return hfiles.toArray(new Path[hfiles.size()]);
   }
 
+  public static String[] listHFileNames(final FileSystem fs, final Path tableDir)
+      throws IOException {
+    Path[] files = listHFiles(fs, tableDir);
+    String[] names = new String[files.length];
+    for (int i = 0; i < files.length; ++i) {
+      names[i] = files[i].getName();
+    }
+    Arrays.sort(names);
+    return names;
+  }
+
   /**
    * Take a snapshot of the specified table and verify that the given family is
    * not empty. Note that this will leave the table disabled
@@ -391,8 +387,7 @@ public class SnapshotTestingUtils {
     }
 
     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), tableName, nonEmptyFamilyNames,
-      emptyFamilyNames, rootDir, admin, fs, false,
-      new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), null);
+      emptyFamilyNames, rootDir, admin, fs);
   }
 
   /**
@@ -414,11 +409,15 @@ public class SnapshotTestingUtils {
     final TableName table = TableName.valueOf(snapshotDesc.getTable());
 
     final ArrayList corruptedFiles = new ArrayList();
-    SnapshotReferenceUtil.visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() {
+    final Configuration conf = util.getConfiguration();
+    SnapshotReferenceUtil.visitTableStoreFiles(conf, fs, snapshotDir, snapshotDesc,
+        new SnapshotReferenceUtil.StoreFileVisitor() {
       @Override
-      public void storeFile (final String region, final String family, final String hfile)
-          throws IOException {
-        HFileLink link = HFileLink.create(util.getConfiguration(), table, region, family, hfile);
+      public void storeFile(final HRegionInfo regionInfo, final String family,
+            final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+        String region = regionInfo.getEncodedName();
+        String hfile = storeFile.getName();
+        HFileLink link = HFileLink.create(conf, table, region, family, hfile);
         if (corruptedFiles.size() % 2 == 0) {
           fs.delete(link.getAvailablePath(fs), true);
           corruptedFiles.add(hfile);
@@ -431,6 +430,191 @@ public class SnapshotTestingUtils {
   }
 
   // ==========================================================================
+  //  Snapshot Mock
+  // ==========================================================================
+  public static class SnapshotMock {
+    private final static String TEST_FAMILY = "cf";
+    public final static int TEST_NUM_REGIONS = 4;
+
+    private final Configuration conf;
+    private final FileSystem fs;
+    private final Path rootDir;
+
+    static class RegionData {
+      public HRegionInfo hri;
+      public Path tableDir;
+      public Path[] files;
+
+      public RegionData(final Path tableDir, final HRegionInfo hri, final int nfiles) {
+        this.tableDir = tableDir;
+        this.hri = hri;
+        this.files = new Path[nfiles];
+      }
+    }
+
+    public static class SnapshotBuilder {
+      private final RegionData[] tableRegions;
+      private final SnapshotDescription desc;
+      private final HTableDescriptor htd;
+      private final Configuration conf;
+      private final FileSystem fs;
+      private final Path rootDir;
+      private Path snapshotDir;
+      private int snapshotted = 0;
+
+      public SnapshotBuilder(final Configuration conf, final FileSystem fs,
+          final Path rootDir, final HTableDescriptor htd,
+          final SnapshotDescription desc, final RegionData[] tableRegions)
+          throws IOException {
+        this.fs = fs;
+        this.conf = conf;
+        this.rootDir = rootDir;
+        this.htd = htd;
+        this.desc = desc;
+        this.tableRegions = tableRegions;
+        this.snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir);
+        new FSTableDescriptors(conf)
+          .createTableDescriptorForTableDirectory(snapshotDir, htd, false);
+      }
+
+      public HTableDescriptor getTableDescriptor() {
+        return this.htd;
+      }
+
+      public SnapshotDescription getSnapshotDescription() {
+        return this.desc;
+      }
+
+      public Path getSnapshotsDir() {
+        return this.snapshotDir;
+      }
+
+      public Path[] addRegion() throws IOException {
+        return addRegion(desc);
+      }
+
+      public Path[] addRegionV1() throws IOException {
+        return addRegion(desc.toBuilder()
+                          .setVersion(SnapshotManifestV1.DESCRIPTOR_VERSION)
+                          .build());
+      }
+
+      public Path[] addRegionV2() throws IOException {
+        return addRegion(desc.toBuilder()
+                          .setVersion(SnapshotManifestV2.DESCRIPTOR_VERSION)
+                          .build());
+      }
+
+      private Path[] addRegion(final SnapshotDescription desc) throws IOException {
+        if (this.snapshotted == tableRegions.length) {
+          throw new UnsupportedOperationException("No more regions in the table");
+        }
+
+        RegionData regionData = tableRegions[this.snapshotted++];
+        ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName());
+        SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor);
+        manifest.addRegion(regionData.tableDir, regionData.hri);
+        return regionData.files;
+      }
+
+      public Path commit() throws IOException {
+        ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName());
+        SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor);
+        manifest.addTableDescriptor(htd);
+        manifest.consolidate();
+        SnapshotDescriptionUtils.completeSnapshot(desc, rootDir, snapshotDir, fs);
+        snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(desc, rootDir);
+        return snapshotDir;
+      }
+    }
+
+    public SnapshotMock(final Configuration conf, final FileSystem fs, final Path rootDir) {
+      this.fs = fs;
+      this.conf = conf;
+      this.rootDir = rootDir;
+    }
+
+    public SnapshotBuilder createSnapshotV1(final String snapshotName) throws IOException {
+      return createSnapshot(snapshotName, SnapshotManifestV1.DESCRIPTOR_VERSION);
+    }
+
+    public SnapshotBuilder createSnapshotV2(final String snapshotName) throws IOException {
+      return createSnapshot(snapshotName, SnapshotManifestV2.DESCRIPTOR_VERSION);
+    }
+
+    private SnapshotBuilder createSnapshot(final String snapshotName, final int version)
+        throws IOException {
+      HTableDescriptor htd = createHtd(snapshotName);
+      htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
+
+      RegionData[] regions = createTable(htd, TEST_NUM_REGIONS);
+
+      SnapshotDescription desc = SnapshotDescription.newBuilder()
+        .setTable(htd.getNameAsString())
+        .setName(snapshotName)
+        .setVersion(version)
+        .build();
+
+      Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir);
+      SnapshotDescriptionUtils.writeSnapshotInfo(desc, workingDir, fs);
+      return new SnapshotBuilder(conf, fs, rootDir, htd, desc, regions);
+    }
+
+    public HTableDescriptor createHtd(final String tableName) {
+      HTableDescriptor htd = new HTableDescriptor(tableName);
+      htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
+      return htd;
+    }
+
+    private RegionData[] createTable(final HTableDescriptor htd, final int nregions)
+        throws IOException {
+      Path tableDir = FSUtils.getTableDir(rootDir, htd.getTableName());
+      new FSTableDescriptors(conf).createTableDescriptorForTableDirectory(tableDir, htd, false);
+
+      assertTrue(nregions % 2 == 0);
+      RegionData[] regions = new RegionData[nregions];
+      for (int i = 0; i < regions.length; i += 2) {
+        byte[] startKey = Bytes.toBytes(0 + i * 2);
+        byte[] endKey = Bytes.toBytes(1 + i * 2);
+
+        // First region, simple with one plain hfile.
+        HRegionInfo hri = new HRegionInfo(htd.getTableName(), startKey, endKey);
+        HRegionFileSystem rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri);
+        regions[i] = new RegionData(tableDir, hri, 3);
+        for (int j = 0; j < regions[i].files.length; ++j) {
+          Path storeFile = createStoreFile(rfs.createTempName());
+          regions[i].files[j] = rfs.commitStoreFile(TEST_FAMILY, storeFile);
+        }
+
+        // Second region, used to test the split case.
+        // This region contains a reference to the hfile in the first region.
+        startKey = Bytes.toBytes(2 + i * 2);
+        endKey = Bytes.toBytes(3 + i * 2);
+        hri = new HRegionInfo(htd.getTableName());
+        rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri);
+        regions[i+1] = new RegionData(tableDir, hri, regions[i].files.length);
+        for (int j = 0; j < regions[i].files.length; ++j) {
+          String refName = regions[i].files[j].getName() + '.' + regions[i].hri.getEncodedName();
+          Path refFile = createStoreFile(new Path(rootDir, refName));
+          regions[i+1].files[j] = rfs.commitStoreFile(TEST_FAMILY, refFile);
+        }
+      }
+      return regions;
+    }
+
+    private Path createStoreFile(final Path storeFile)
+        throws IOException {
+      FSDataOutputStream out = fs.create(storeFile);
+      try {
+        out.write(Bytes.toBytes(storeFile.toString()));
+      } finally {
+        out.close();
+      }
+      return storeFile;
+    }
+  }
+
+  // ==========================================================================
   //  Table Helpers
   // ==========================================================================
   public static void waitForTableToBeOnline(final HBaseTestingUtility util,

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java?rev=1593139&r1=1593138&r2=1593139&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java Wed May  7 21:28:12 2014
@@ -24,7 +24,6 @@ import static org.junit.Assert.assertTru
 import java.io.IOException;
 import java.net.URI;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
@@ -48,9 +47,12 @@ import org.apache.hadoop.hbase.client.HB
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo;
+import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.FSTableDescriptors;
 import org.apache.hadoop.hbase.util.Pair;
 import org.junit.After;
 import org.junit.AfterClass;
@@ -143,9 +145,13 @@ public class TestExportSnapshot {
   @Test
   public void testBalanceSplit() throws Exception {
     // Create a list of files
-    List<Pair<Path, Long>> files = new ArrayList<Pair<Path, Long>>();
+    List<Pair<SnapshotFileInfo, Long>> files = new ArrayList<Pair<SnapshotFileInfo, Long>>();
     for (long i = 0; i <= 20; i++) {
-      files.add(new Pair<Path, Long>(new Path("file-" + i), i));
+      SnapshotFileInfo fileInfo = SnapshotFileInfo.newBuilder()
+        .setType(SnapshotFileInfo.Type.HFILE)
+        .setHfile("file-" + i)
+        .build();
+      files.add(new Pair<SnapshotFileInfo, Long>(fileInfo, i));
     }
 
     // Create 5 groups (total size 210)
@@ -154,18 +160,26 @@ public class TestExportSnapshot {
     //    group 2: 18, 13,  8,  3 (total size: 42)
     //    group 3: 17, 12,  7,  4 (total size: 42)
     //    group 4: 16, 11,  6,  5 (total size: 42)
-    List<List<Path>> splits = ExportSnapshot.getBalancedSplits(files, 5);
+    List<List<SnapshotFileInfo>> splits = ExportSnapshot.getBalancedSplits(files, 5);
     assertEquals(5, splits.size());
-    assertEquals(Arrays.asList(new Path("file-20"), new Path("file-11"),
-      new Path("file-10"), new Path("file-1"), new Path("file-0")), splits.get(0));
-    assertEquals(Arrays.asList(new Path("file-19"), new Path("file-12"),
-      new Path("file-9"), new Path("file-2")), splits.get(1));
-    assertEquals(Arrays.asList(new Path("file-18"), new Path("file-13"),
-      new Path("file-8"), new Path("file-3")), splits.get(2));
-    assertEquals(Arrays.asList(new Path("file-17"), new Path("file-14"),
-      new Path("file-7"), new Path("file-4")), splits.get(3));
-    assertEquals(Arrays.asList(new Path("file-16"), new Path("file-15"),
-      new Path("file-6"), new Path("file-5")), splits.get(4));
+
+    String[] split0 = new String[] {"file-20", "file-11", "file-10", "file-1", "file-0"};
+    verifyBalanceSplit(splits.get(0), split0);
+    String[] split1 = new String[] {"file-19", "file-12", "file-9",  "file-2"};
+    verifyBalanceSplit(splits.get(1), split1);
+    String[] split2 = new String[] {"file-18", "file-13", "file-8",  "file-3"};
+    verifyBalanceSplit(splits.get(2), split2);
+    String[] split3 = new String[] {"file-17", "file-14", "file-7",  "file-4"};
+    verifyBalanceSplit(splits.get(3), split3);
+    String[] split4 = new String[] {"file-16", "file-15", "file-6",  "file-5"};
+    verifyBalanceSplit(splits.get(4), split4);
+  }
+
+  private void verifyBalanceSplit(final List<SnapshotFileInfo> split, final String[] expected) {
+    assertEquals(expected.length, split.size());
+    for (int i = 0; i < expected.length; ++i) {
+      assertEquals(expected[i], split.get(i).getHfile());
+    }
   }
 
   /**
@@ -233,6 +247,11 @@ public class TestExportSnapshot {
     r1fs.commitStoreFile(TEST_FAMILY, storeFile);
 
     Path tableDir = FSUtils.getTableDir(archiveDir, tableWithRefsName);
+    HTableDescriptor htd = new HTableDescriptor(tableWithRefsName);
+    htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
+    new FSTableDescriptors(fs, rootDir)
+        .createTableDescriptorForTableDirectory(tableDir, htd, false);
+
     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
     FileUtil.copy(fs, tableDir, fs, snapshotDir, false, conf);
     SnapshotDescriptionUtils.writeSnapshotInfo(sd, snapshotDir, fs);
@@ -345,13 +364,14 @@ public class TestExportSnapshot {
       new Path(HConstants.SNAPSHOT_DIR_NAME, snapshotName));
     final Path exportedArchive = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
     LOG.debug(listFiles(fs, exportedArchive, exportedArchive));
-    SnapshotReferenceUtil.visitReferencedFiles(fs, exportedSnapshot,
-        new SnapshotReferenceUtil.FileVisitor() {
-        public void storeFile (final String region, final String family, final String hfile)
-            throws IOException {
+    SnapshotReferenceUtil.visitReferencedFiles(TEST_UTIL.getConfiguration(), fs, exportedSnapshot,
+          new SnapshotReferenceUtil.SnapshotVisitor() {
+        public void storeFile(final HRegionInfo regionInfo, final String family,
+            final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+          String hfile = storeFile.getName();
           verifyNonEmptyFile(new Path(exportedArchive,
             new Path(FSUtils.getTableDir(new Path("./"), tableName),
-                new Path(region, new Path(family, hfile)))));
+                new Path(regionInfo.getEncodedName(), new Path(family, hfile)))));
         }
 
         public void recoveredEdits (final String region, final String logfile)

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java?rev=1593139&r1=1593138&r2=1593139&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java Wed May  7 21:28:12 2014
@@ -40,7 +40,6 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.LargeTests;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.TableNotFoundException;
@@ -53,10 +52,8 @@ import org.apache.hadoop.hbase.master.HM
 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
 import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSTableDescriptors;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
 import org.apache.log4j.Level;
@@ -153,15 +150,6 @@ public class TestFlushSnapshotFromClient
     HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
     SnapshotTestingUtils.loadData(UTIL, table, DEFAULT_NUM_ROWS, TEST_FAM);
 
-    // get the name of all the regionservers hosting the snapshotted table
-    Set<String> snapshotServers = new HashSet<String>();
-    List<RegionServerThread> servers = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads();
-    for (RegionServerThread server : servers) {
-      if (server.getRegionServer().getOnlineRegions(TABLE_NAME).size() > 0) {
-        snapshotServers.add(server.getRegionServer().getServerName().toString());
-      }
-    }
-
     LOG.debug("FS state before snapshot:");
     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
         FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
@@ -184,7 +172,7 @@ public class TestFlushSnapshotFromClient
         FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
 
     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
-        admin, fs, false, new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), snapshotServers);
+        admin, fs);
   }
 
 
@@ -202,15 +190,6 @@ public class TestFlushSnapshotFromClient
     HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
     SnapshotTestingUtils.loadData(UTIL, table, DEFAULT_NUM_ROWS, TEST_FAM);
 
-    // get the name of all the regionservers hosting the snapshotted table
-    Set<String> snapshotServers = new HashSet<String>();
-    List<RegionServerThread> servers = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads();
-    for (RegionServerThread server : servers) {
-      if (server.getRegionServer().getOnlineRegions(TABLE_NAME).size() > 0) {
-        snapshotServers.add(server.getRegionServer().getServerName().toString());
-      }
-    }
-
     LOG.debug("FS state before snapshot:");
     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
         FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
@@ -238,7 +217,7 @@ public class TestFlushSnapshotFromClient
         FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
 
     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
-        admin, fs, false, new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), snapshotServers);
+        admin, fs);
   }
 
   @Test (timeout=300000)
@@ -399,46 +378,11 @@ public class TestFlushSnapshotFromClient
     SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
 
     String snapshotName = "flushSnapshotCreateListDestroy";
-    // test creating the snapshot
-    admin.snapshot(snapshotName, STRING_TABLE_NAME, SnapshotDescription.Type.FLUSH);
-    logFSTree(FSUtils.getRootDir(UTIL.getConfiguration()));
-
-    // make sure we only have 1 matching snapshot
-    List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
-      snapshotName, TABLE_NAME);
-
-    // check the directory structure
     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
-    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshots.get(0), rootDir);
-    assertTrue(fs.exists(snapshotDir));
-    FSUtils.logFileSystemState(UTIL.getTestFileSystem(), snapshotDir, LOG);
-    Path snapshotinfo = new Path(snapshotDir, SnapshotDescriptionUtils.SNAPSHOTINFO_FILE);
-    assertTrue(fs.exists(snapshotinfo));
-
-    // check the table info
-    HTableDescriptor desc = FSTableDescriptors.getTableDescriptorFromFs(fs,
-        rootDir, TABLE_NAME);
-    HTableDescriptor snapshotDesc = FSTableDescriptors.getTableDescriptorFromFs(fs,
-        new Path(SnapshotDescriptionUtils.getSnapshotsDir(rootDir), snapshotName));
-    assertEquals(desc, snapshotDesc);
-
-    // check the region snapshot for all the regions
-    List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME);
-    assertTrue(regions.size() > 1);
-    for (HRegionInfo info : regions) {
-      String regionName = info.getEncodedName();
-      Path regionDir = new Path(snapshotDir, regionName);
-      HRegionInfo snapshotRegionInfo = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
-      assertEquals(info, snapshotRegionInfo);
-      // check to make sure we have the family
-      Path familyDir = new Path(regionDir, Bytes.toString(TEST_FAM));
-      assertTrue("Missing region " + Bytes.toString(snapshotRegionInfo.getStartKey()),
-                 fs.exists(familyDir));
-
-      // make sure we have some file references
-      assertTrue(fs.listStatus(familyDir).length > 0);
-    }
+    SnapshotTestingUtils.createSnapshotAndValidate(admin,
+      TableName.valueOf(STRING_TABLE_NAME), Bytes.toString(TEST_FAM),
+      snapshotName, rootDir, fs, true);
   }
 
   /**



Mime
View raw message