hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From te...@apache.org
Subject hbase git commit: HBASE-19551 hbck -boundaries doesn't work correctly (Toshihiro Suzuki)
Date Sun, 31 Dec 2017 07:18:19 GMT
Repository: hbase
Updated Branches:
  refs/heads/branch-1 528eb1082 -> e866e837f


HBASE-19551 hbck -boundaries doesn't work correctly (Toshihiro Suzuki)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e866e837
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e866e837
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e866e837

Branch: refs/heads/branch-1
Commit: e866e837fb70f079778556951aa92b633345aea9
Parents: 528eb10
Author: tedyu <yuzhihong@gmail.com>
Authored: Sat Dec 30 23:18:12 2017 -0800
Committer: tedyu <yuzhihong@gmail.com>
Committed: Sat Dec 30 23:18:12 2017 -0800

----------------------------------------------------------------------
 .../org/apache/hadoop/hbase/util/HBaseFsck.java | 22 +++---
 .../apache/hadoop/hbase/util/TestHBaseFsck.java | 71 ++++++++++++++++++--
 .../hadoop/hbase/util/hbck/HbckTestingUtil.java |  8 +++
 3 files changed, 85 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/e866e837/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 066f2b3..d0fa17d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -872,15 +872,17 @@ public class HBaseFsck extends Configured implements Closeable {
             for (FileStatus storeFile : storeFiles) {
               HFile.Reader reader = HFile.createReader(fs, storeFile.getPath(), new CacheConfig(
                   getConf()), getConf());
-              if ((reader.getFirstKey() != null)
-                  && ((storeFirstKey == null) || (comparator.compare(storeFirstKey,
-                      reader.getFirstKey()) > 0))) {
-                storeFirstKey = reader.getFirstKey();
+              if (reader.getFirstKey() != null) {
+                byte[] firstKey = keyOnly(reader.getFirstKey());
+                if (storeFirstKey == null || comparator.compare(storeFirstKey, firstKey)
> 0) {
+                  storeFirstKey = firstKey;
+                }
               }
-              if ((reader.getLastKey() != null)
-                  && ((storeLastKey == null) || (comparator.compare(storeLastKey,
-                      reader.getLastKey())) < 0)) {
-                storeLastKey = reader.getLastKey();
+              if (reader.getLastKey() != null) {
+                byte[] lastKey = keyOnly(reader.getLastKey());
+                if (storeLastKey == null || comparator.compare(storeLastKey, lastKey) <
0) {
+                  storeLastKey = lastKey;
+                }
               }
               reader.close();
             }
@@ -888,8 +890,8 @@ public class HBaseFsck extends Configured implements Closeable {
         }
         currentRegionBoundariesInformation.metaFirstKey = regionInfo.getStartKey();
         currentRegionBoundariesInformation.metaLastKey = regionInfo.getEndKey();
-        currentRegionBoundariesInformation.storesFirstKey = keyOnly(storeFirstKey);
-        currentRegionBoundariesInformation.storesLastKey = keyOnly(storeLastKey);
+        currentRegionBoundariesInformation.storesFirstKey = storeFirstKey;
+        currentRegionBoundariesInformation.storesLastKey = storeLastKey;
         if (currentRegionBoundariesInformation.metaFirstKey.length == 0)
           currentRegionBoundariesInformation.metaFirstKey = null;
         if (currentRegionBoundariesInformation.metaLastKey.length == 0)

http://git-wip-us.apache.org/repos/asf/hbase/blob/e866e837/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
index 6859a11..38985de 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.util;
 
 import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.assertErrors;
 import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.assertNoErrors;
+import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.checkRegionBoundaries;
 import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.doFsck;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -3063,15 +3064,73 @@ public class TestHBaseFsck {
 
   @Test (timeout = 180000)
   public void testRegionBoundariesCheck() throws Exception {
-    HBaseFsck hbck = doFsck(conf, false);
+    TableName tableName = TableName.valueOf("testRegionBoundariesCheck");
+
+    // setup a table
+    HTableDescriptor desc = new HTableDescriptor(tableName);
+    HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM));
+    desc.addFamily(hcd); // If a table has no CF's it doesn't get checked
+    createTable(TEST_UTIL, desc, SPLITS);
+
+    Table table = connection.getTable(tableName, tableExecutorService);
+    List<Put> puts = new ArrayList<>();
+
+    // for the first region
+    puts.add(new Put(Bytes.toBytes("0")).addColumn(FAM, Bytes.toBytes("col"),
+      Bytes.toBytes("val")));
+    puts.add(new Put(Bytes.toBytes("999")).addColumn(FAM, Bytes.toBytes("col"),
+      Bytes.toBytes("val")));
+
+    // for the second region
+    puts.add(new Put(Bytes.toBytes("AA")).addColumn(FAM, Bytes.toBytes("col"),
+      Bytes.toBytes("val")));
+    puts.add(new Put(Bytes.toBytes("AZ")).addColumn(FAM, Bytes.toBytes("col"),
+      Bytes.toBytes("val")));
+
+    table.put(puts);
+
+    // to guarantee all data flushed, disable and enable the table
+    admin.disableTable(tableName);
+    admin.enableTable(tableName);
+
+    // check region boundaries before moving an HFile
+    HBaseFsck hbck = checkRegionBoundaries(conf);
     assertNoErrors(hbck); // no errors
-    try {
-      hbck.checkRegionBoundaries();
-    } catch (IllegalArgumentException e) {
-      if (e.getMessage().endsWith("not a valid DFS filename.")) {
-        fail("Table directory path is not valid." + e.getMessage());
+
+    // move an HFile in the second region to the first region directory
+    admin.disableTable(tableName);
+
+    List<HRegionInfo> tableRegions = admin.getTableRegions(tableName);
+    HRegionInfo firstRegion = tableRegions.get(0);
+    HRegionInfo secondRegion = tableRegions.get(1);
+
+    FileSystem fs = FileSystem.get(conf);
+    Path tableDir= FSUtils.getTableDir(FSUtils.getRootDir(conf), tableName);
+    Path firstRegionFamDir = new Path(new Path(tableDir, firstRegion.getEncodedName()), FAM_STR);
+    Path hfileInFirstRegion = getHFilePath(fs, firstRegionFamDir);
+    Path secondRegionFamDir = new Path(new Path(tableDir, secondRegion.getEncodedName()),
FAM_STR);
+    Path hfileInSecondRegion = getHFilePath(fs, secondRegionFamDir);
+
+    // rename HFile names (to "0" and "1") in order to guarantee the same file iteration
order of
+    // fs.listStatus()
+    fs.rename(hfileInFirstRegion, new Path(firstRegionFamDir, "0"));
+    fs.rename(hfileInSecondRegion, new Path(firstRegionFamDir, "1"));
+
+    admin.enableTable(tableName);
+
+    // check region boundaries after moving an HFile
+    hbck = checkRegionBoundaries(conf);
+    assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.BOUNDARIES_ERROR });
+  }
+
+  private static Path getHFilePath(FileSystem fs, Path famDir) throws IOException {
+    FileStatus[] hfFss = fs.listStatus(famDir);
+    for (FileStatus hfs : hfFss) {
+      if (!hfs.isDirectory()) {
+        return hfs.getPath();
       }
     }
+    return null;
   }
 
   @org.junit.Rule

http://git-wip-us.apache.org/repos/asf/hbase/blob/e866e837/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
index 1807add..37f900f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
@@ -88,6 +88,14 @@ public class HbckTestingUtil {
     return hbck;
   }
 
+  public static HBaseFsck checkRegionBoundaries(Configuration conf) throws Exception {
+    HBaseFsck hbck = new HBaseFsck(conf, exec);
+    hbck.connect();
+    hbck.checkRegionBoundaries();
+    hbck.close();
+    return hbck;
+  }
+
   public static boolean inconsistencyFound(HBaseFsck fsck) throws Exception {
     List<ERROR_CODE> errs = fsck.getErrors().getErrorList();
     return (errs != null && !errs.isEmpty());


Mime
View raw message