hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e...@apache.org
Subject hbase git commit: HBASE-14302 TableSnapshotInputFormat should not create back references when restoring snapshot
Date Tue, 25 Aug 2015 20:48:29 GMT
Repository: hbase
Updated Branches:
  refs/heads/branch-1.0 9134d70f2 -> 6e4dd5b80


HBASE-14302 TableSnapshotInputFormat should not create back references when restoring snapshot

Conflicts:
	hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
	hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java
	hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6e4dd5b8
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6e4dd5b8
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6e4dd5b8

Branch: refs/heads/branch-1.0
Commit: 6e4dd5b80f5f72b78fb067bdcaec170439a003e4
Parents: 9134d70
Author: Enis Soztutar <enis@apache.org>
Authored: Tue Aug 25 12:10:15 2015 -0700
Committer: Enis Soztutar <enis@apache.org>
Committed: Tue Aug 25 13:45:35 2015 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/hbase/io/HFileLink.java   | 89 +++++++++++++++++---
 .../hbase/snapshot/RestoreSnapshotHelper.java   | 35 ++++++--
 .../mapred/TestTableSnapshotInputFormat.java    |  9 ++
 .../TableSnapshotInputFormatTestBase.java       | 51 +++++++++++
 .../mapreduce/TestTableSnapshotInputFormat.java | 10 +++
 5 files changed, 175 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/6e4dd5b8/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java
index 2ef59d1..ba357f9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java
@@ -291,9 +291,30 @@ public class HFileLink extends FileLink {
   public static boolean create(final Configuration conf, final FileSystem fs,
       final Path dstFamilyPath, final HRegionInfo hfileRegionInfo,
       final String hfileName) throws IOException {
+    return create(conf, fs, dstFamilyPath, hfileRegionInfo, hfileName, true);
+  }
+
+  /**
+   * Create a new HFileLink
+   *
+   * <p>It also adds a back-reference to the hfile back-reference directory
+   * to simplify the reference-count and the cleaning process.
+   *
+   * @param conf {@link Configuration} to read for the archive directory name
+   * @param fs {@link FileSystem} on which to write the HFileLink
+   * @param dstFamilyPath - Destination path (table/region/cf/)
+   * @param hfileRegionInfo - Linked HFile Region Info
+   * @param hfileName - Linked HFile name
+   * @param createBackRef - Whether back reference should be created. Defaults to true.
+   * @return true if the file is created, otherwise the file exists.
+   * @throws IOException on file or parent directory creation failure
+   */
+  public static boolean create(final Configuration conf, final FileSystem fs,
+      final Path dstFamilyPath, final HRegionInfo hfileRegionInfo,
+      final String hfileName, final boolean createBackRef) throws IOException {
     TableName linkedTable = hfileRegionInfo.getTable();
     String linkedRegion = hfileRegionInfo.getEncodedName();
-    return create(conf, fs, dstFamilyPath, linkedTable, linkedRegion, hfileName);
+    return create(conf, fs, dstFamilyPath, linkedTable, linkedRegion, hfileName, createBackRef);
   }
 
   /**
@@ -314,6 +335,28 @@ public class HFileLink extends FileLink {
   public static boolean create(final Configuration conf, final FileSystem fs,
       final Path dstFamilyPath, final TableName linkedTable, final String linkedRegion,
       final String hfileName) throws IOException {
+    return create(conf, fs, dstFamilyPath, linkedTable, linkedRegion, hfileName, true);
+  }
+
+  /**
+   * Create a new HFileLink
+   *
+   * <p>It also adds a back-reference to the hfile back-reference directory
+   * to simplify the reference-count and the cleaning process.
+   *
+   * @param conf {@link Configuration} to read for the archive directory name
+   * @param fs {@link FileSystem} on which to write the HFileLink
+   * @param dstFamilyPath - Destination path (table/region/cf/)
+   * @param linkedTable - Linked Table Name
+   * @param linkedRegion - Linked Region Name
+   * @param hfileName - Linked HFile name
+   * @param createBackRef - Whether back reference should be created. Defaults to true.
+   * @return true if the file is created, otherwise the file exists.
+   * @throws IOException on file or parent directory creation failure
+   */
+  public static boolean create(final Configuration conf, final FileSystem fs,
+      final Path dstFamilyPath, final TableName linkedTable, final String linkedRegion,
+      final String hfileName, final boolean createBackRef) throws IOException {
     String familyName = dstFamilyPath.getName();
     String regionName = dstFamilyPath.getParent().getName();
     String tableName = FSUtils.getTableName(dstFamilyPath.getParent().getParent())
@@ -328,19 +371,24 @@ public class HFileLink extends FileLink {
     // Make sure the FileLink reference directory exists
     Path archiveStoreDir = HFileArchiveUtil.getStoreArchivePath(conf,
           linkedTable, linkedRegion, familyName);
-    Path backRefssDir = getBackReferencesDir(archiveStoreDir, hfileName);
-    fs.mkdirs(backRefssDir);
-
-    // Create the reference for the link
-    Path backRefPath = new Path(backRefssDir, refName);
-    fs.createNewFile(backRefPath);
+    Path backRefPath = null;
+    if (createBackRef) {
+      Path backRefssDir = getBackReferencesDir(archiveStoreDir, hfileName);
+      fs.mkdirs(backRefssDir);
+
+      // Create the reference for the link
+      backRefPath = new Path(backRefssDir, refName);
+      fs.createNewFile(backRefPath);
+    }
     try {
       // Create the link
       return fs.createNewFile(new Path(dstFamilyPath, name));
     } catch (IOException e) {
       LOG.error("couldn't create the link=" + name + " for " + dstFamilyPath, e);
       // Revert the reference if the link creation failed
-      fs.delete(backRefPath, false);
+      if (createBackRef) {
+        fs.delete(backRefPath, false);
+      }
       throw e;
     }
   }
@@ -359,13 +407,34 @@ public class HFileLink extends FileLink {
    * @throws IOException on file or parent directory creation failure
    */
   public static boolean createFromHFileLink(final Configuration conf, final FileSystem fs,
-      final Path dstFamilyPath, final String hfileLinkName) throws IOException {
+      final Path dstFamilyPath, final String hfileLinkName)
+          throws IOException {
+    return createFromHFileLink(conf, fs, dstFamilyPath, hfileLinkName, true);
+  }
+
+  /**
+   * Create a new HFileLink starting from a hfileLink name
+   *
+   * <p>It also adds a back-reference to the hfile back-reference directory
+   * to simplify the reference-count and the cleaning process.
+   *
+   * @param conf {@link Configuration} to read for the archive directory name
+   * @param fs {@link FileSystem} on which to write the HFileLink
+   * @param dstFamilyPath - Destination path (table/region/cf/)
+   * @param hfileLinkName - HFileLink name (it contains hfile-region-table)
+   * @param createBackRef - Whether back reference should be created. Defaults to true.
+   * @return true if the file is created, otherwise the file exists.
+   * @throws IOException on file or parent directory creation failure
+   */
+  public static boolean createFromHFileLink(final Configuration conf, final FileSystem fs,
+      final Path dstFamilyPath, final String hfileLinkName, final boolean createBackRef)
+          throws IOException {
     Matcher m = LINK_NAME_PATTERN.matcher(hfileLinkName);
     if (!m.matches()) {
       throw new IllegalArgumentException(hfileLinkName + " is not a valid HFileLink name!");
     }
     return create(conf, fs, dstFamilyPath, TableName.valueOf(m.group(1), m.group(2)),
-        m.group(3), m.group(4));
+        m.group(3), m.group(4), createBackRef);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hbase/blob/6e4dd5b8/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
index f28125e..d192ade 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
@@ -127,6 +127,7 @@ public class RestoreSnapshotHelper {
 
   private final Configuration conf;
   private final FileSystem fs;
+  private final boolean createBackRefs;
 
   public RestoreSnapshotHelper(final Configuration conf,
       final FileSystem fs,
@@ -134,7 +135,18 @@ public class RestoreSnapshotHelper {
       final HTableDescriptor tableDescriptor,
       final Path rootDir,
       final ForeignExceptionDispatcher monitor,
-      final MonitoredTask status)
+      final MonitoredTask status) {
+    this(conf, fs, manifest, tableDescriptor, rootDir, monitor, status, true);
+  }
+
+  public RestoreSnapshotHelper(final Configuration conf,
+      final FileSystem fs,
+      final SnapshotManifest manifest,
+      final HTableDescriptor tableDescriptor,
+      final Path rootDir,
+      final ForeignExceptionDispatcher monitor,
+      final MonitoredTask status,
+      final boolean createBackRefs)
   {
     this.fs = fs;
     this.conf = conf;
@@ -146,6 +158,7 @@ public class RestoreSnapshotHelper {
     this.tableDir = FSUtils.getTableDir(rootDir, tableDesc.getTableName());
     this.monitor = monitor;
     this.status = status;
+    this.createBackRefs = createBackRefs;
   }
 
   /**
@@ -434,7 +447,7 @@ public class RestoreSnapshotHelper {
         for (SnapshotRegionManifest.StoreFile storeFile: hfilesToAdd) {
           LOG.debug("Adding HFileLink " + storeFile.getName() +
             " to region=" + regionInfo.getEncodedName() + " table=" + tableName);
-          restoreStoreFile(familyDir, regionInfo, storeFile);
+          restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs);
         }
       } else {
         // Family doesn't exists in the snapshot
@@ -455,7 +468,7 @@ public class RestoreSnapshotHelper {
 
       for (SnapshotRegionManifest.StoreFile storeFile: familyEntry.getValue()) {
         LOG.trace("Adding HFileLink " + storeFile.getName() + " to table=" + tableName);
-        restoreStoreFile(familyDir, regionInfo, storeFile);
+        restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs);
       }
     }
   }
@@ -538,7 +551,7 @@ public class RestoreSnapshotHelper {
       Path familyDir = new Path(regionDir, familyFiles.getFamilyName().toStringUtf8());
       for (SnapshotRegionManifest.StoreFile storeFile: familyFiles.getStoreFilesList()) {
         LOG.info("Adding HFileLink " + storeFile.getName() + " to table=" + tableName);
-        restoreStoreFile(familyDir, snapshotRegionInfo, storeFile);
+        restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs);
       }
     }
   }
@@ -553,17 +566,19 @@ public class RestoreSnapshotHelper {
    * </ul>
    * @param familyDir destination directory for the store file
    * @param regionInfo destination region info for the table
-   * @param hfileName store file name (can be a Reference, HFileLink or simple HFile)
+   * @param storeFile store file name (can be a Reference, HFileLink or simple HFile)
+   * @param createBackRef - Whether back reference should be created. Defaults to true.
    */
   private void restoreStoreFile(final Path familyDir, final HRegionInfo regionInfo,
-      final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+      final SnapshotRegionManifest.StoreFile storeFile, final boolean createBackRef)
+          throws IOException {
     String hfileName = storeFile.getName();
     if (HFileLink.isHFileLink(hfileName)) {
-      HFileLink.createFromHFileLink(conf, fs, familyDir, hfileName);
+      HFileLink.createFromHFileLink(conf, fs, familyDir, hfileName, createBackRef);
     } else if (StoreFileInfo.isReference(hfileName)) {
       restoreReferenceFile(familyDir, regionInfo, storeFile);
     } else {
-      HFileLink.create(conf, fs, familyDir, regionInfo, hfileName);
+      HFileLink.create(conf, fs, familyDir, regionInfo, hfileName, createBackRef);
     }
   }
 
@@ -729,8 +744,10 @@ public class RestoreSnapshotHelper {
         "Restoring  snapshot '" + snapshotName + "' to directory " + restoreDir);
     ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher();
 
+    // we send createBackRefs=false so that restored hfiles do not create back reference
links
+    // in the base hbase root dir.
     RestoreSnapshotHelper helper = new RestoreSnapshotHelper(conf, fs,
-      manifest, manifest.getTableDescriptor(), restoreDir, monitor, status);
+      manifest, manifest.getTableDescriptor(), restoreDir, monitor, status, false);
     helper.restoreHdfsRegions(); // TODO: parallelize.
 
     if (LOG.isDebugEnabled()) {

http://git-wip-us.apache.org/repos/asf/hbase/blob/6e4dd5b8/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableSnapshotInputFormat.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableSnapshotInputFormat.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableSnapshotInputFormat.java
index 7707c19..29c37c5 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableSnapshotInputFormat.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableSnapshotInputFormat.java
@@ -148,6 +148,15 @@ public class TestTableSnapshotInputFormat extends TableSnapshotInputFormatTestBa
   }
 
   @Override
+  public void testRestoreSnapshotDoesNotCreateBackRefLinksInit(TableName tableName,
+      String snapshotName, Path tmpTableDir) throws Exception {
+    JobConf job = new JobConf(UTIL.getConfiguration());
+    TableMapReduceUtil.initTableSnapshotMapJob(snapshotName,
+      COLUMNS, TestTableSnapshotMapper.class, ImmutableBytesWritable.class,
+      NullWritable.class, job, false, tmpTableDir);
+  }
+
+  @Override
   protected void testWithMockedMapReduce(HBaseTestingUtility util, String snapshotName,
       int numRegions, int expectedNumSplits) throws Exception {
     setupCluster();

http://git-wip-us.apache.org/repos/asf/hbase/blob/6e4dd5b8/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
index 1f8ccae..b35a5d3 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.hbase.mapreduce;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.Cell;
@@ -28,14 +29,20 @@ import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.Admin;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.io.HFileLink;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
+import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
 import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.HFileArchiveUtil;
 import org.junit.Assert;
 import org.junit.Test;
 
+import static org.junit.Assert.assertFalse;
+
 import java.io.IOException;
 import java.util.Arrays;
 
@@ -101,6 +108,50 @@ public abstract class TableSnapshotInputFormatTestBase {
     testWithMapReduce(UTIL, "testWithMapReduceAndOfflineHBaseMultiRegion", 10, 8, true);
   }
 
+  // Test that snapshot restore does not create back references in the HBase root dir.
+  @Test
+  public void testRestoreSnapshotDoesNotCreateBackRefLinks() throws Exception {
+    setupCluster();
+    TableName tableName = TableName.valueOf("testRestoreSnapshotDoesNotCreateBackRefLinks");
+    String snapshotName = "foo";
+
+    try {
+      createTableAndSnapshot(UTIL, tableName, snapshotName, getStartRow(), getEndRow(), 1);
+
+      Path tmpTableDir = UTIL.getDataTestDirOnTestFS(snapshotName);
+
+      testRestoreSnapshotDoesNotCreateBackRefLinksInit(tableName, snapshotName,tmpTableDir);
+
+      Path rootDir = FSUtils.getRootDir(UTIL.getConfiguration());
+      for (Path regionDir : FSUtils.getRegionDirs(fs, FSUtils.getTableDir(rootDir, tableName)))
{
+        for (Path storeDir : FSUtils.getFamilyDirs(fs, regionDir)) {
+          for (FileStatus status : fs.listStatus(storeDir)) {
+            System.out.println(status.getPath());
+            if (StoreFileInfo.isValid(status)) {
+              Path archiveStoreDir = HFileArchiveUtil.getStoreArchivePath(UTIL.getConfiguration(),
+                tableName, regionDir.getName(), storeDir.getName());
+
+              Path path = HFileLink.getBackReferencesDir(storeDir, status.getPath().getName());
+              // assert back references directory is empty
+              assertFalse("There is a back reference in " + path, fs.exists(path));
+
+              path = HFileLink.getBackReferencesDir(archiveStoreDir, status.getPath().getName());
+              // assert back references directory is empty
+              assertFalse("There is a back reference in " + path, fs.exists(path));
+            }
+          }
+        }
+      }
+    } finally {
+      UTIL.getHBaseAdmin().deleteSnapshot(snapshotName);
+      UTIL.deleteTable(tableName);
+      tearDownCluster();
+    }
+  }
+
+  public abstract void testRestoreSnapshotDoesNotCreateBackRefLinksInit(TableName tableName,
+      String snapshotName, Path tmpTableDir) throws Exception;
+
   protected void testWithMapReduce(HBaseTestingUtility util, String snapshotName,
       int numRegions, int expectedNumSplits, boolean shutdownCluster) throws Exception {
     setupCluster();

http://git-wip-us.apache.org/repos/asf/hbase/blob/6e4dd5b8/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java
index 62fae77..2846128 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java
@@ -172,6 +172,16 @@ public class TestTableSnapshotInputFormat extends TableSnapshotInputFormatTestBa
     }
   }
 
+  @Override
+  public void testRestoreSnapshotDoesNotCreateBackRefLinksInit(TableName tableName,
+      String snapshotName, Path tmpTableDir) throws Exception {
+    Job job = new Job(UTIL.getConfiguration());
+    TableMapReduceUtil.initTableSnapshotMapperJob(snapshotName,
+      new Scan(), TestTableSnapshotMapper.class, ImmutableBytesWritable.class,
+      NullWritable.class, job, false, tmpTableDir);
+  }
+
+  @Override
   public void testWithMockedMapReduce(HBaseTestingUtility util, String snapshotName,
       int numRegions, int expectedNumSplits) throws Exception {
     setupCluster();


Mime
View raw message