hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mberto...@apache.org
Subject svn commit: r1527021 - in /hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase: master/cleaner/TestSnapshotFromMaster.java snapshot/SnapshotTestingUtils.java
Date Fri, 27 Sep 2013 18:27:30 GMT
Author: mbertozzi
Date: Fri Sep 27 18:27:30 2013
New Revision: 1527021

URL: http://svn.apache.org/r1527021
Log:
HBASE-9548 Cleanup SnapshotTestingUtils

Modified:
    hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
    hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java

Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java?rev=1527021&r1=1527020&r2=1527021&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
(original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
Fri Sep 27 18:27:30 2013
@@ -304,19 +304,19 @@ public class TestSnapshotFromMaster {
 
     // get the snapshot files for the table
     Path snapshotTable = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
-    FileStatus[] snapshotHFiles = SnapshotTestingUtils.listHFiles(fs, snapshotTable);
+    Path[] snapshotHFiles = SnapshotTestingUtils.listHFiles(fs, snapshotTable);
     // check that the files in the archive contain the ones that we need for the snapshot
     LOG.debug("Have snapshot hfiles:");
-    for (FileStatus file : snapshotHFiles) {
-      LOG.debug(file.getPath());
+    for (Path file : snapshotHFiles) {
+      LOG.debug(file);
     }
     // get the archived files for the table
     Collection<String> files = getArchivedHFiles(archiveDir, rootDir, fs, STRING_TABLE_NAME);
 
     // and make sure that there is a proper subset
-    for (FileStatus file : snapshotHFiles) {
-      assertTrue("Archived hfiles " + files + " is missing snapshot file:" + file.getPath(),
-        files.contains(file.getPath().getName()));
+    for (Path file : snapshotHFiles) {
+      assertTrue("Archived hfiles " + files + " is missing snapshot file:" + file,
+        files.contains(file.getName()));
     }
 
     // delete the existing snapshot
@@ -350,12 +350,12 @@ public class TestSnapshotFromMaster {
   private final Collection<String> getArchivedHFiles(Path archiveDir, Path rootDir,
       FileSystem fs, String tableName) throws IOException {
     Path tableArchive = new Path(archiveDir, tableName);
-    FileStatus[] archivedHFiles = SnapshotTestingUtils.listHFiles(fs, tableArchive);
+    Path[] archivedHFiles = SnapshotTestingUtils.listHFiles(fs, tableArchive);
     List<String> files = new ArrayList<String>(archivedHFiles.length);
     LOG.debug("Have archived hfiles: " + tableArchive);
-    for (FileStatus file : archivedHFiles) {
-      LOG.debug(file.getPath());
-      files.add(file.getPath().getName());
+    for (Path file : archivedHFiles) {
+      LOG.debug(file);
+      files.add(file.getName());
     }
     // sort the archived files
 

Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java?rev=1527021&r1=1527020&r2=1527021&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
(original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
Fri Sep 27 18:27:30 2013
@@ -18,6 +18,7 @@
 package org.apache.hadoop.hbase.snapshot;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
@@ -26,6 +27,7 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
 import java.util.Set;
+import java.util.HashSet;
 import java.util.TreeSet;
 
 import org.apache.commons.logging.Log;
@@ -39,6 +41,7 @@ import org.apache.hadoop.hbase.HColumnDe
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableNotEnabledException;
 import org.apache.hadoop.hbase.client.Durability;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
@@ -48,12 +51,10 @@ import org.apache.hadoop.hbase.master.Ma
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
-import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException;
-import org.apache.hadoop.hbase.snapshot.HSnapshotDescription;
-import org.apache.hadoop.hbase.snapshot.TakeSnapshotUtils;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSTableDescriptors;
 import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.FSVisitor;
 import org.apache.hadoop.hbase.util.MD5Hash;
 import org.junit.Assert;
 
@@ -73,12 +74,34 @@ public class SnapshotTestingUtils {
   }
 
   /**
-   * Make sure that there is only one snapshot returned from the master and its name and
table match
-   * the passed in parameters.
+   * Make sure that there is only one snapshot returned from the master and its
+   * name and table match the passed in parameters.
    */
-  public static void assertOneSnapshotThatMatches(HBaseAdmin admin, HSnapshotDescription
snapshot)
+  public static List<SnapshotDescription> assertExistsMatchingSnapshot(
+      HBaseAdmin admin, String snapshotName, String tableName)
       throws IOException {
-    assertOneSnapshotThatMatches(admin, snapshot.getName(), snapshot.getTable());
+    // list the snapshot
+    List<SnapshotDescription> snapshots = admin.listSnapshots();
+
+    List<SnapshotDescription> returnedSnapshots = new ArrayList<SnapshotDescription>();
+    for (SnapshotDescription sd : snapshots) {
+      if (snapshotName.equals(sd.getName()) &&
+          tableName.equals(sd.getTable())) {
+        returnedSnapshots.add(sd);
+      }
+    }
+
+    Assert.assertTrue("No matching snapshots found.", returnedSnapshots.size()>0);
+    return returnedSnapshots;
+  }
+
+  /**
+   * Make sure that there is only one snapshot returned from the master
+   */
+  public static void assertOneSnapshotThatMatches(HBaseAdmin admin,
+      HSnapshotDescription snapshot) throws IOException {
+    assertOneSnapshotThatMatches(admin, snapshot.getName(),
+        snapshot.getTable());
   }
 
   /**
@@ -91,11 +114,12 @@ public class SnapshotTestingUtils {
   }
 
   /**
-   * Make sure that there is only one snapshot returned from the master and its name and
table match
-   * the passed in parameters.
+   * Make sure that there is only one snapshot returned from the master and its
+   * name and table match the passed in parameters.
    */
-  public static List<SnapshotDescription> assertOneSnapshotThatMatches(HBaseAdmin admin,
-      String snapshotName, String tableName) throws IOException {
+  public static List<SnapshotDescription> assertOneSnapshotThatMatches(
+      HBaseAdmin admin, String snapshotName, String tableName)
+      throws IOException {
     // list the snapshot
     List<SnapshotDescription> snapshots = admin.listSnapshots();
 
@@ -107,47 +131,122 @@ public class SnapshotTestingUtils {
   }
 
   /**
-   * Make sure that there is only one snapshot returned from the master and its name and
table match
-   * the passed in parameters.
+   * Make sure that there is only one snapshot returned from the master and its
+   * name and table match the passed in parameters.
+   */
+  public static List<SnapshotDescription> assertOneSnapshotThatMatches(
+      HBaseAdmin admin, byte[] snapshot, byte[] tableName) throws IOException {
+    return assertOneSnapshotThatMatches(admin, Bytes.toString(snapshot),
+        Bytes.toString(tableName));
+  }
+
+  /**
+   * Confirm that the snapshot contains references to all the files that should
+   * be in the snapshot.
    */
-  public static List<SnapshotDescription> assertOneSnapshotThatMatches(HBaseAdmin admin,
-      byte[] snapshot, byte[] tableName) throws IOException {
-    return assertOneSnapshotThatMatches(admin, Bytes.toString(snapshot), Bytes.toString(tableName));
+  public static void confirmSnapshotValid(
+      SnapshotDescription snapshotDescriptor, byte[] tableName,
+      byte[] testFamily, Path rootDir, HBaseAdmin admin, FileSystem fs,
+      boolean requireLogs, Path logsDir, Set<String> snapshotServers)
+      throws IOException {
+    ArrayList nonEmptyTestFamilies = new ArrayList(1);
+    nonEmptyTestFamilies.add(testFamily);
+    confirmSnapshotValid(snapshotDescriptor, Bytes.toString(tableName),
+      nonEmptyTestFamilies, null, rootDir, admin, fs, requireLogs,
+      logsDir, snapshotServers);
+  }
+
+  /**
+   * Confirm that the snapshot has no references files but only metadata.
+   */
+  public static void confirmEmptySnapshotValid(
+      SnapshotDescription snapshotDescriptor, byte[] tableName,
+      byte[] testFamily, Path rootDir, HBaseAdmin admin, FileSystem fs,
+      boolean requireLogs, Path logsDir, Set<String> snapshotServers)
+      throws IOException {
+    ArrayList emptyTestFamilies = new ArrayList(1);
+    emptyTestFamilies.add(testFamily);
+    confirmSnapshotValid(snapshotDescriptor, Bytes.toString(tableName),
+      null, emptyTestFamilies, rootDir, admin, fs, requireLogs,
+      logsDir, snapshotServers);
   }
 
   /**
-   * Confirm that the snapshot contains references to all the files that should be in the
snapshot
+   * Confirm that the snapshot contains references to all the files that should
+   * be in the snapshot. This method also perform some redundant check like
+   * the existence of the snapshotinfo or the regioninfo which are done always
+   * by the MasterSnapshotVerifier, at the end of the snapshot operation.
    */
-  public static void confirmSnapshotValid(SnapshotDescription snapshotDescriptor,
-      byte[] tableName, byte[] testFamily, Path rootDir, HBaseAdmin admin, FileSystem fs,
-      boolean requireLogs, Path logsDir, Set<String> snapshotServers) throws IOException
{
-    Path snapshotDir = SnapshotDescriptionUtils
-        .getCompletedSnapshotDir(snapshotDescriptor, rootDir);
+  public static void confirmSnapshotValid(
+      SnapshotDescription snapshotDescriptor, String tableName,
+      List<byte[]> nonEmptyTestFamilies, List<byte[]> emptyTestFamilies,
+      Path rootDir, HBaseAdmin admin, FileSystem fs, boolean requireLogs,
+      Path logsDir, Set<String> snapshotServers) throws IOException {
+    // check snapshot dir
+    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
+        snapshotDescriptor, rootDir);
     assertTrue(fs.exists(snapshotDir));
+
+    // check snapshot info
     Path snapshotinfo = new Path(snapshotDir, SnapshotDescriptionUtils.SNAPSHOTINFO_FILE);
     assertTrue(fs.exists(snapshotinfo));
+
     // check the logs dir
     if (requireLogs) {
-      TakeSnapshotUtils.verifyAllLogsGotReferenced(fs, logsDir, snapshotServers,
-        snapshotDescriptor, new Path(snapshotDir, HConstants.HREGION_LOGDIR_NAME));
+      TakeSnapshotUtils.verifyAllLogsGotReferenced(fs, logsDir,
+          snapshotServers, snapshotDescriptor, new Path(snapshotDir,
+              HConstants.HREGION_LOGDIR_NAME));
     }
+
     // check the table info
-    HTableDescriptor desc = FSTableDescriptors.getTableDescriptor(fs, rootDir, tableName);
-    HTableDescriptor snapshotDesc = FSTableDescriptors.getTableDescriptor(fs, snapshotDir);
+    HTableDescriptor desc = FSTableDescriptors.getTableDescriptorFromFs(fs, rootDir, tableName);
+    HTableDescriptor snapshotDesc = FSTableDescriptors.getTableDescriptorFromFs(fs, snapshotDir);
     assertEquals(desc, snapshotDesc);
 
+    // Extract regions and families with store files
+    final Set<String> snapshotRegions = new HashSet<String>();
+    final Set<byte[]> snapshotFamilies = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
+    FSVisitor.visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() {
+      public void storeFile(final String region, final String family, final String hfileName)
+          throws IOException {
+        snapshotRegions.add(region);
+        snapshotFamilies.add(Bytes.toBytes(family));
+      }
+    });
+
+    // Verify that there are store files in the specified families
+    if (nonEmptyTestFamilies != null) {
+      for (final byte[] familyName: nonEmptyTestFamilies) {
+        assertTrue(snapshotFamilies.contains(familyName));
+      }
+    }
+
+    // Verify that there are no store files in the specified families
+    if (emptyTestFamilies != null) {
+      for (final byte[] familyName: emptyTestFamilies) {
+        assertFalse(snapshotFamilies.contains(familyName));
+      }
+    }
+
+    // Avoid checking regions if the request is for an empty snapshot
+    if ((nonEmptyTestFamilies == null || nonEmptyTestFamilies.size() == 0) &&
+        (emptyTestFamilies != null && emptyTestFamilies.size() > 0)) {
+      assertEquals(0, snapshotRegions.size());
+      return;
+    }
+
     // check the region snapshot for all the regions
-    List<HRegionInfo> regions = admin.getTableRegions(tableName);
+    List<HRegionInfo> regions = admin.getTableRegions(Bytes.toBytes(tableName));
+    assertEquals(regions.size(), snapshotRegions.size());
+
+    // Verify Regions
     for (HRegionInfo info : regions) {
       String regionName = info.getEncodedName();
+      assertTrue(snapshotRegions.contains(regionName));
+
       Path regionDir = new Path(snapshotDir, regionName);
       HRegionInfo snapshotRegionInfo = HRegion.loadDotRegionInfoFileContent(fs, regionDir);
       assertEquals(info, snapshotRegionInfo);
-      // check to make sure we have the family
-      Path familyDir = new Path(regionDir, Bytes.toString(testFamily));
-      assertTrue("Expected to find: " + familyDir + ", but it doesn't exist", fs.exists(familyDir));
-      // make sure we have some files references
-      assertTrue(fs.listStatus(familyDir).length > 0);
     }
   }
 
@@ -172,11 +271,13 @@ public class SnapshotTestingUtils {
     }
   }
 
-  public static void cleanupSnapshot(HBaseAdmin admin, byte[] tableName) throws IOException
{
+  public static void cleanupSnapshot(HBaseAdmin admin, byte[] tableName)
+      throws IOException {
     SnapshotTestingUtils.cleanupSnapshot(admin, Bytes.toString(tableName));
   }
 
-  public static void cleanupSnapshot(HBaseAdmin admin, String snapshotName) throws IOException
{
+  public static void cleanupSnapshot(HBaseAdmin admin, String snapshotName)
+      throws IOException {
     // delete the taken snapshot
     admin.deleteSnapshot(snapshotName);
     assertNoSnapshots(admin);
@@ -202,63 +303,66 @@ public class SnapshotTestingUtils {
 
   /**
    * List all the HFiles in the given table
-   * @param fs FileSystem where the table lives
+   *
+   * @param fs: FileSystem where the table lives
    * @param tableDir directory of the table
    * @return array of the current HFiles in the table (could be a zero-length array)
    * @throws IOException on unexecpted error reading the FS
    */
-  public static FileStatus[] listHFiles(final FileSystem fs, Path tableDir) throws IOException
{
-    // setup the filters we will need based on the filesystem
-    PathFilter regionFilter = new FSUtils.RegionDirFilter(fs);
-    PathFilter familyFilter = new FSUtils.FamilyDirFilter(fs);
-    final PathFilter fileFilter = new PathFilter() {
-      @Override
-      public boolean accept(Path file) {
-        try {
-          return fs.isFile(file);
-        } catch (IOException e) {
-          return false;
-        }
+  public static Path[] listHFiles(final FileSystem fs, final Path tableDir)
+      throws IOException {
+    final ArrayList<Path> hfiles = new ArrayList<Path>();
+    FSVisitor.visitTableStoreFiles(fs, tableDir, new FSVisitor.StoreFileVisitor() {
+      public void storeFile(final String region, final String family, final String hfileName)
+          throws IOException {
+        hfiles.add(new Path(tableDir, new Path(region, new Path(family, hfileName))));
       }
-    };
+    });
+    return hfiles.toArray(new Path[hfiles.size()]);
+  }
 
-    FileStatus[] regionDirs = FSUtils.listStatus(fs, tableDir, regionFilter);
-    // if no regions, then we are done
-    if (regionDirs == null || regionDirs.length == 0) return new FileStatus[0];
-
-    // go through each of the regions, and add al the hfiles under each family
-    List<FileStatus> regionFiles = new ArrayList<FileStatus>(regionDirs.length);
-    for (FileStatus regionDir : regionDirs) {
-      FileStatus[] fams = FSUtils.listStatus(fs, regionDir.getPath(), familyFilter);
-      // if no families, then we are done again
-      if (fams == null || fams.length == 0) continue;
-      // add all the hfiles under the family
-      regionFiles.addAll(SnapshotTestingUtils.getHFilesInRegion(fams, fs, fileFilter));
-    }
-    FileStatus[] files = new FileStatus[regionFiles.size()];
-    regionFiles.toArray(files);
-    return files;
+  /**
+   * Take a snapshot of the specified table and verify that the given family is
+   * not empty. Note that this will leave the table disabled
+   * in the case of an offline snapshot.
+   */
+  public static void createSnapshotAndValidate(HBaseAdmin admin,
+      String tableName, String familyName, String snapshotNameString,
+      Path rootDir, FileSystem fs, boolean onlineSnapshot)
+      throws Exception {
+    ArrayList<byte[]> nonEmptyFamilyNames = new ArrayList<byte[]>(1);
+    nonEmptyFamilyNames.add(Bytes.toBytes(familyName));
+    createSnapshotAndValidate(admin, tableName, nonEmptyFamilyNames, /* emptyFamilyNames=
*/ null,
+                              snapshotNameString, rootDir, fs, onlineSnapshot);
   }
 
   /**
-   * Get all the hfiles in the region, under the passed set of families
-   * @param families all the family directories under the region
-   * @param fs filesystem where the families live
-   * @param fileFilter filter to only include files
-   * @return collection of all the hfiles under all the passed in families (non-null)
-   * @throws IOException on unexecpted error reading the FS
+   * Take a snapshot of the specified table and verify the given families.
+   * Note that this will leave the table disabled in the case of an offline snapshot.
    */
-  public static Collection<FileStatus> getHFilesInRegion(FileStatus[] families, FileSystem
fs,
-      PathFilter fileFilter) throws IOException {
-    Set<FileStatus> files = new TreeSet<FileStatus>();
-    for (FileStatus family : families) {
-      // get all the hfiles in the family
-      FileStatus[] hfiles = FSUtils.listStatus(fs, family.getPath(), fileFilter);
-      // if no hfiles, then we are done with this family
-      if (hfiles == null || hfiles.length == 0) continue;
-      files.addAll(Arrays.asList(hfiles));
+  public static void createSnapshotAndValidate(HBaseAdmin admin,
+      String tableName, List<byte[]> nonEmptyFamilyNames, List<byte[]> emptyFamilyNames,
+      String snapshotNameString, Path rootDir, FileSystem fs, boolean onlineSnapshot)
+        throws Exception {
+    if (!onlineSnapshot) {
+      try {
+        admin.disableTable(tableName);
+      } catch (TableNotEnabledException tne) {
+        LOG.info("In attempting to disable " + tableName + " it turns out that the this table
is " +
+            "already disabled.");
+      }
     }
-    return files;
+    admin.snapshot(snapshotNameString, tableName);
+
+    List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertExistsMatchingSnapshot(admin,
+      snapshotNameString, tableName);
+    if (snapshots == null || snapshots.size() != 1) {
+      Assert.fail("Incorrect number of snapshots for table " + tableName);
+    }
+
+    SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), tableName, nonEmptyFamilyNames,
+      emptyFamilyNames, rootDir, admin, fs, false,
+      new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), null);
   }
 
   // ==========================================================================



Mime
View raw message