hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From chenh...@apache.org
Subject hbase git commit: HBASE-16464 archive folder grows bigger and bigger due to corrupt snapshot under tmp dir
Date Wed, 24 Aug 2016 05:49:56 GMT
Repository: hbase
Updated Branches:
  refs/heads/branch-1.2 71e583df3 -> 09ade3208


HBASE-16464 archive folder grows bigger and bigger due to corrupt snapshot under tmp dir


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/09ade320
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/09ade320
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/09ade320

Branch: refs/heads/branch-1.2
Commit: 09ade32086cdb3ba76587da773de007c17bdfc5c
Parents: 71e583d
Author: chenheng <chenheng@apache.org>
Authored: Mon Aug 22 16:45:20 2016 +0800
Committer: chenheng <chenheng@apache.org>
Committed: Wed Aug 24 13:48:14 2016 +0800

----------------------------------------------------------------------
 .../master/snapshot/SnapshotFileCache.java      | 17 ++++++++++++++++-
 .../master/snapshot/TakeSnapshotHandler.java    |  2 +-
 .../snapshot/SnapshotDescriptionUtils.java      | 14 ++++++++++++++
 .../hbase/snapshot/SnapshotManifestV2.java      | 18 ++++++++++++------
 .../snapshot/TestSnapshotHFileCleaner.java      | 20 ++++++++++++++++++++
 .../hbase/snapshot/SnapshotTestingUtils.java    | 12 ++++++++++++
 6 files changed, 75 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/09ade320/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
index dfd3cb5..5b367c5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException;
 import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
 import org.apache.hadoop.hbase.util.FSUtils;
 
@@ -300,7 +301,21 @@ public class SnapshotFileCache implements Stoppable {
     FileStatus[] running = FSUtils.listStatus(fs, snapshotTmpDir);
     if (running != null) {
       for (FileStatus run : running) {
-        snapshotInProgress.addAll(fileInspector.filesUnderSnapshot(run.getPath()));
+        try {
+          snapshotInProgress.addAll(fileInspector.filesUnderSnapshot(run.getPath()));
+        } catch (CorruptedSnapshotException e) {
+          // See HBASE-16464
+          if (e.getCause() instanceof FileNotFoundException) {
+            // If the snapshot is not in progress, we will delete it
+            if (!fs.exists(new Path(run.getPath(),
+              SnapshotDescriptionUtils.SNAPSHOT_IN_PROGRESS))) {
+              fs.delete(run.getPath(), true);
+              LOG.warn("delete the " + run.getPath() + " due to exception:", e.getCause());
+            }
+          } else {
+            throw e;
+          }
+        }
       }
     }
     return snapshotInProgress;

http://git-wip-us.apache.org/repos/asf/hbase/blob/09ade320/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
index 7c87ea1..39387cb 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
@@ -164,7 +164,7 @@ public abstract class TakeSnapshotHandler extends EventHandler implements
Snapsh
     try {
       // If regions move after this meta scan, the region specific snapshot should fail,
triggering
       // an external exception that gets captured here.
-
+      SnapshotDescriptionUtils.createInProgressTag(workingDir, fs);
       // write down the snapshot info in the working directory
       SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, workingDir, fs);
       snapshotManifest.addTableDescriptor(this.htd);

http://git-wip-us.apache.org/repos/asf/hbase/blob/09ade320/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
index b6b608c..aebb647 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hbase.snapshot;
 
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.Collections;
 
@@ -103,6 +104,9 @@ public final class SnapshotDescriptionUtils {
 
   /** Temporary directory under the snapshot directory to store in-progress snapshots */
   public static final String SNAPSHOT_TMP_DIR_NAME = ".tmp";
+
+  /** This tag will be created in in-progess snapshots */
+  public static final String SNAPSHOT_IN_PROGRESS = ".inprogress";
   // snapshot operation values
   /** Default value if no start time is specified */
   public static final long NO_SNAPSHOT_START_TIME_SPECIFIED = 0;
@@ -292,6 +296,16 @@ public final class SnapshotDescriptionUtils {
   }
 
   /**
+   * Create in-progress tag under .tmp of in-progress snapshot
+   * */
+  public static void createInProgressTag(Path workingDir, FileSystem fs) throws IOException
{
+    FsPermission perms = FSUtils.getFilePermissions(fs, fs.getConf(),
+      HConstants.DATA_FILE_UMASK_KEY);
+    Path snapshot_in_progress = new Path(workingDir, SnapshotDescriptionUtils.SNAPSHOT_IN_PROGRESS);
+    FSUtils.create(fs, snapshot_in_progress, perms, true);
+  }
+
+  /**
    * Read in the {@link org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription}
stored for the snapshot in the passed directory
    * @param fs filesystem where the snapshot was taken
    * @param snapshotDir directory where the snapshot was stored

http://git-wip-us.apache.org/repos/asf/hbase/blob/09ade320/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV2.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV2.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV2.java
index 0f65f18..933f9d9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV2.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV2.java
@@ -80,12 +80,18 @@ public class SnapshotManifestV2 {
     }
 
     public void regionClose(final SnapshotRegionManifest.Builder region) throws IOException
{
-      SnapshotRegionManifest manifest = region.build();
-      FSDataOutputStream stream = fs.create(getRegionManifestPath(snapshotDir, manifest));
-      try {
-        manifest.writeTo(stream);
-      } finally {
-        stream.close();
+      // we should ensure the snapshot dir exist, maybe it has been deleted by master
+      // see HBASE-16464
+      if (fs.exists(snapshotDir)) {
+        SnapshotRegionManifest manifest = region.build();
+        FSDataOutputStream stream = fs.create(getRegionManifestPath(snapshotDir, manifest));
+        try {
+          manifest.writeTo(stream);
+        } finally {
+          stream.close();
+        }
+      } else {
+        LOG.warn("can't write manifest without parent dir, maybe it has been deleted by master?");
       }
     }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/09ade320/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotHFileCleaner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotHFileCleaner.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotHFileCleaner.java
index 616907c..78be717 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotHFileCleaner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotHFileCleaner.java
@@ -166,4 +166,24 @@ public class TestSnapshotHFileCleaner {
       fs.delete(SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir), true);
     }
   }
+
+
+  /**
+   * HBASE-16464
+   */
+  @Test
+  public void testMissedTmpSnapshot() throws IOException {
+    SnapshotTestingUtils.SnapshotMock
+      snapshotMock = new SnapshotTestingUtils.SnapshotMock(TEST_UTIL.getConfiguration(),
fs, rootDir);
+    SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2(
+      SNAPSHOT_NAME_STR, TABLE_NAME_STR);
+    builder.addRegionV2();
+    builder.missOneRegionSnapshotFile();
+
+    long period = Long.MAX_VALUE;
+    SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
+      "test-snapshot-file-cache-refresh", new SnapshotFiles());
+    cache.getSnapshotsInProgress();
+    assertFalse(fs.exists(builder.getSnapshotsDir()));
+  }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/09ade320/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
index f6eb62a..1e67718 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
@@ -572,6 +572,18 @@ public final class SnapshotTestingUtils {
         corruptFile(p);
       }
 
+      public void missOneRegionSnapshotFile() throws IOException {
+        FileStatus[] manifestFiles = FSUtils.listStatus(fs, snapshotDir);
+        for (FileStatus fileStatus : manifestFiles) {
+          String fileName = fileStatus.getPath().getName();
+          if (fileName.endsWith(SnapshotDescriptionUtils.SNAPSHOTINFO_FILE)
+            || fileName.endsWith(".tabledesc")
+            || fileName.endsWith(SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME)) {
+              fs.delete(fileStatus.getPath(), true);
+          }
+        }
+      }
+
       /**
        * Corrupt data-manifest file
        *


Mime
View raw message