hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mberto...@apache.org
Subject [1/4] hbase git commit: HBASE-13227 LoadIncrementalHFile should skip non-files inside a possible family-dir
Date Fri, 13 Mar 2015 17:23:06 GMT
Repository: hbase
Updated Branches:
  refs/heads/0.98 41f865378 -> 2f831915d
  refs/heads/branch-1 aa93b176f -> 7229a0a47
  refs/heads/branch-1.0 7a508a39a -> 3c8bb9424
  refs/heads/master 0f69ea409 -> d2d4ef14f


HBASE-13227 LoadIncrementalHFile should skip non-files inside a possible family-dir


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7229a0a4
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7229a0a4
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7229a0a4

Branch: refs/heads/branch-1
Commit: 7229a0a47df97fdc418308be17d2d91f2d534fcd
Parents: aa93b17
Author: Matteo Bertozzi <matteo.bertozzi@cloudera.com>
Authored: Fri Mar 13 17:11:27 2015 +0000
Committer: Matteo Bertozzi <matteo.bertozzi@cloudera.com>
Committed: Fri Mar 13 17:12:11 2015 +0000

----------------------------------------------------------------------
 .../hbase/mapreduce/LoadIncrementalHFiles.java      | 16 ++++++++++++++--
 .../hbase/mapreduce/TestLoadIncrementalHFiles.java  |  6 ++++--
 2 files changed, 18 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/7229a0a4/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
index 9c120c2..53efc42 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
@@ -209,6 +209,10 @@ public class LoadIncrementalHFiles extends Configured implements Tool
{
       byte[] family = familyDir.getName().getBytes();
       FileStatus[] hfileStatuses = fs.listStatus(familyDir);
       for (FileStatus hfileStatus : hfileStatuses) {
+        if (!hfileStatus.isFile()) {
+          LOG.warn("Skipping non-file " + hfileStatus);
+          continue;
+        }
         long length = hfileStatus.getLen();
         Path hfile = hfileStatus.getPath();
         // Skip "_", reference, HFileLink
@@ -293,10 +297,18 @@ public class LoadIncrementalHFiles extends Configured implements Tool
{
         LoadQueueItem lqi = queueIter.next();
         String familyNameInHFile = Bytes.toString(lqi.family);
         if (!familyNames.contains(familyNameInHFile)) {
-          if (HFile.isHFileFormat(lqi.hfilePath.getFileSystem(getConf()), lqi.hfilePath))
{
+          boolean isValid = false;
+          try {
+            isValid = HFile.isHFileFormat(lqi.hfilePath.getFileSystem(getConf()), lqi.hfilePath);
+            if (!isValid) {
+              LOG.warn("the file " + lqi + " doesn't seems to be an hfile. skipping");
+            }
+          } catch (FileNotFoundException e) {
+            LOG.warn("the file " + lqi + " was removed");
+          }
+          if (isValid) {
             unmatchedFamilies.add(familyNameInHFile);
           } else {
-            LOG.warn("the file " + lqi + " doesn't seems to be an hfile. skipping");
             queueIter.remove();
           }
         }

http://git-wip-us.apache.org/repos/asf/hbase/blob/7229a0a4/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
index 3d05782..7cc57f5 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
@@ -308,8 +308,9 @@ public class TestLoadIncrementalHFiles {
   }
 
   /**
-   * Write a random data file in a dir with a valid family name but not part of the table
families
-   * we should we able to bulkload without getting the unmatched family exception. HBASE-13037
+   * Write a random data file and a non-file in a dir with a valid family name
+   * but not part of the table families. we should we able to bulkload without
+   * getting the unmatched family exception. HBASE-13037/HBASE-13227
    */
   @Test(timeout = 60000)
   public void testNonHfileFolderWithUnmatchedFamilyName() throws Exception {
@@ -324,6 +325,7 @@ public class TestLoadIncrementalHFiles {
     final String NON_FAMILY_FOLDER = "_logs";
     Path nonFamilyDir = new Path(dir, NON_FAMILY_FOLDER);
     fs.mkdirs(nonFamilyDir);
+    fs.mkdirs(new Path(nonFamilyDir, "non-file"));
     createRandomDataFile(fs, new Path(nonFamilyDir, "012356789"), 16 * 1024);
 
     Table table = null;


Mime
View raw message