hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jxi...@apache.org
Subject svn commit: r1347868 - in /hbase/trunk/hbase-server/src: main/java/org/apache/hadoop/hbase/util/HBaseFsck.java test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
Date Fri, 08 Jun 2012 02:32:14 GMT
Author: jxiang
Date: Fri Jun  8 02:32:14 2012
New Revision: 1347868

URL: http://svn.apache.org/viewvc?rev=1347868&view=rev
Log:
HBASE-6173 hbck check specified tables only

Modified:
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java?rev=1347868&r1=1347867&r2=1347868&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java Fri
Jun  8 02:32:14 2012
@@ -145,7 +145,6 @@ public class HBaseFsck {
   public static final long DEFAULT_TIME_LAG = 60000; // default value of 1 minute
   public static final long DEFAULT_SLEEP_BEFORE_RERUN = 10000;
   private static final int MAX_NUM_THREADS = 50; // #threads to contact regions
-  private static final long THREADS_KEEP_ALIVE_SECONDS = 60;
   private static boolean rsSupportsOffline = true;
   private static final int DEFAULT_OVERLAPS_TO_SIDELINE = 2;
   private static final int DEFAULT_MAX_MERGE = 5;
@@ -174,8 +173,9 @@ public class HBaseFsck {
   private boolean fixHdfsOrphans = false; // fix fs holes (missing .regioninfo)
   private boolean fixVersionFile = false; // fix missing hbase.version file in hdfs
 
-  // limit fixes to listed tables, if empty atttempt to fix all
-  private List<byte[]> tablesToFix = new ArrayList<byte[]>();
+  // limit checking/fixes to listed tables, if empty attempt to check/fix all
+  // -ROOT- and .META. are always checked
+  private Set<String> tablesIncluded = new HashSet<String>();
   private int maxMerge = DEFAULT_MAX_MERGE; // maximum number of overlapping regions to merge
   private int maxOverlapsToSideline = DEFAULT_OVERLAPS_TO_SIDELINE; // maximum number of
overlapping regions to sideline
   private boolean sidelineBigOverlaps = false; // sideline overlaps with >maxMerge regions
@@ -205,6 +205,11 @@ public class HBaseFsck {
    * This map from Tablename -> TableInfo contains the structures necessary to
    * detect table consistency problems (holes, dupes, overlaps).  It is sorted
    * to prevent dupes.
+   *
+   * If tablesIncluded is empty, this map contains all tables.
+   * Otherwise, it contains only meta tables and tables in tablesIncluded,
+   * unless checkMetaOnly is specified, in which case, it contains only
+   * the meta tables (.META. and -ROOT-).
    */
   private SortedMap<String, TableInfo> tablesInfo = new ConcurrentSkipListMap<String,TableInfo>();
 
@@ -670,6 +675,7 @@ public class HBaseFsck {
         // only executed once per table.
         modTInfo = new TableInfo(tableName);
         Path hbaseRoot = new Path(conf.get(HConstants.HBASE_DIR));
+        tablesInfo.put(tableName, modTInfo);
         try {
           HTableDescriptor htd =
               FSTableDescriptors.getTableDescriptor(hbaseRoot.getFileSystem(conf),
@@ -679,10 +685,8 @@ public class HBaseFsck {
           LOG.error("Unable to read .tableinfo from " + hbaseRoot, ioe);
           throw ioe;
         }
-
       }
       modTInfo.addRegionInfo(hbi);
-      tablesInfo.put(tableName, modTInfo);
     }
 
     return tablesInfo;
@@ -835,14 +839,8 @@ public class HBaseFsck {
     for (TableInfo tInfo : tablesInfo.values()) {
       TableIntegrityErrorHandler handler;
       if (fixHoles || fixOverlaps) {
-        if (shouldFixTable(Bytes.toBytes(tInfo.getName()))) {
-          handler = tInfo.new HDFSIntegrityFixer(tInfo, errors, conf,
-              fixHoles, fixOverlaps);
-        } else {
-          LOG.info("Table " + tInfo.getName() + " is not in the include table " +
-            "list.  Just suggesting fixes.");
-          handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
-        }
+        handler = tInfo.new HDFSIntegrityFixer(tInfo, errors, conf,
+          fixHoles, fixOverlaps);
       } else {
         handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
       }
@@ -1032,7 +1030,7 @@ public class HBaseFsck {
       if (dirName.equals(HConstants.VERSION_FILE_NAME)) {
         foundVersionFile = true;
       } else {
-        if (!checkMetaOnly ||
+        if ((!checkMetaOnly && isTableIncluded(dirName)) ||
             dirName.equals("-ROOT-") ||
             dirName.equals(".META.")) {
           tableDirs.add(file);
@@ -2215,9 +2213,14 @@ public class HBaseFsck {
           if (pair.getSecond() != null) {
             sn = pair.getSecond();
           }
-          MetaEntry m = new MetaEntry(pair.getFirst(), sn, ts);
+          HRegionInfo hri = pair.getFirst();
+          if (!(isTableIncluded(hri.getTableNameAsString())
+              || hri.isMetaRegion() || hri.isRootRegion())) {
+            return true;
+          }
+          MetaEntry m = new MetaEntry(hri, sn, ts);
           HbckInfo hbInfo = new HbckInfo(m);
-          HbckInfo previous = regionInfoMap.put(pair.getFirst().getEncodedName(), hbInfo);
+          HbckInfo previous = regionInfoMap.put(hri.getEncodedName(), hbInfo);
           if (previous != null) {
             throw new IOException("Two entries in META are same " + previous);
           }
@@ -2243,7 +2246,7 @@ public class HBaseFsck {
       // Scan .META. to pick up user regions
       MetaScanner.metaScan(conf, visitor);
     }
-    
+
     errors.print("");
     return true;
   }
@@ -2649,9 +2652,8 @@ public class HBaseFsck {
 
         // list all online regions from this region server
         List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(server);
-        if (hbck.checkMetaOnly) {
-          regions = filterOnlyMetaRegions(regions);
-        }
+        regions = filterRegions(regions);
+
         if (details) {
           errors.detail("RegionServer: " + rsinfo.getServerName() +
                            " number of regions: " + regions.size());
@@ -2677,10 +2679,11 @@ public class HBaseFsck {
       return null;
     }
 
-    private List<HRegionInfo> filterOnlyMetaRegions(List<HRegionInfo> regions)
{
+    private List<HRegionInfo> filterRegions(List<HRegionInfo> regions) {
       List<HRegionInfo> ret = Lists.newArrayList();
       for (HRegionInfo hri : regions) {
-        if (hri.isMetaTable()) {
+        if (hri.isMetaTable() || (!hbck.checkMetaOnly
+            && hbck.isTableIncluded(hri.getTableNameAsString()))) {
           ret.add(hri);
         }
       }
@@ -2925,22 +2928,15 @@ public class HBaseFsck {
   }
 
   /**
-   * Only fix tables specified by the list
+   * Only check/fix tables specified by the list,
+   * Empty list means all tables are included.
    */
-  boolean shouldFixTable(byte[] table) {
-    if (tablesToFix.size() == 0) {
-      return true;
-    }
-
-    // doing this naively since byte[] equals may not do what we want.
-    for (byte[] t : tablesToFix) {
-      if (Bytes.equals(t, table)) return true;
-    }
-    return false;
+  boolean isTableIncluded(String table) {
+    return (tablesIncluded.size() == 0) || tablesIncluded.contains(table);
   }
 
-  void includeTable(byte[] table) {
-    tablesToFix.add(table);
+  public void includeTable(String table) {
+    tablesIncluded.add(table);
   }
 
   /**
@@ -3102,9 +3098,8 @@ public class HBaseFsck {
         System.err.println("Unrecognized option:" + cmd);
         printUsageAndExit();
       } else {
-        byte[] table = Bytes.toBytes(cmd);
-        fsck.includeTable(table);
-        System.out.println("Allow fixes for table: " + cmd);
+        fsck.includeTable(cmd);
+        System.out.println("Allow checking/fixes for table: " + cmd);
       }
     }
     // do the real work of fsck

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java?rev=1347868&r1=1347867&r2=1347868&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
(original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
Fri Jun  8 02:32:14 2012
@@ -968,6 +968,51 @@ public class TestHBaseFsck {
     }
   }
 
+  /**
+   * This creates two tables and mess both of them and fix them one by one
+   */
+  @Test
+  public void testFixByTable() throws Exception {
+    String table1 = "testFixByTable1";
+    String table2 = "testFixByTable2";
+    try {
+      setupTable(table1);
+      // make sure data in regions, if in hlog only there is no data loss
+      TEST_UTIL.getHBaseAdmin().flush(table1);
+      // Mess them up by leaving a hole in the hdfs data
+      deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
+        Bytes.toBytes("C"), false, false, true); // don't rm meta
+
+      setupTable(table2);
+      // make sure data in regions, if in hlog only there is no data loss
+      TEST_UTIL.getHBaseAdmin().flush(table2);
+      // Mess them up by leaving a hole in the hdfs data
+      deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
+        Bytes.toBytes("C"), false, false, true); // don't rm meta
+
+      HBaseFsck hbck = doFsck(conf, false);
+      assertErrors(hbck, new ERROR_CODE[] {
+        ERROR_CODE.NOT_IN_HDFS, ERROR_CODE.NOT_IN_HDFS});
+
+      // fix hole in table 1
+      doFsck(conf, true, table1);
+      // check that hole in table 1 fixed
+      assertNoErrors(doFsck(conf, false, table1));
+      // check that hole in table 2 still there
+      assertErrors(doFsck(conf, false, table2),
+        new ERROR_CODE[] {ERROR_CODE.NOT_IN_HDFS});
+
+      // fix hole in table 2
+      doFsck(conf, true, table2);
+      // check that hole in both tables fixed
+      assertNoErrors(doFsck(conf, false));
+      assertEquals(ROWKEYS.length - 2, countRows());
+    } finally {
+      deleteTable(table1);
+      deleteTable(table2);
+    }
+  }
+
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java?rev=1347868&r1=1347867&r2=1347868&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
(original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
Fri Jun  8 02:32:14 2012
@@ -28,13 +28,20 @@ import org.apache.hadoop.hbase.util.HBas
 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
 
 public class HbckTestingUtil {
-  public static HBaseFsck doFsck(Configuration conf, boolean fix) throws Exception {
-    return doFsck(conf, fix, fix, fix, fix,fix, fix);
+  public static HBaseFsck doFsck(
+      Configuration conf, boolean fix) throws Exception {
+    return doFsck(conf, fix, null);
+  }
+
+  public static HBaseFsck doFsck(
+      Configuration conf, boolean fix, String table) throws Exception {
+    return doFsck(conf, fix, fix, fix, fix,fix, fix, table);
   }
 
   public static HBaseFsck doFsck(Configuration conf, boolean fixAssignments,
       boolean fixMeta, boolean fixHdfsHoles, boolean fixHdfsOverlaps,
-      boolean fixHdfsOrphans, boolean fixVersionFile) throws Exception {
+      boolean fixHdfsOrphans, boolean fixVersionFile,
+      String table) throws Exception {
     HBaseFsck fsck = new HBaseFsck(conf);
     fsck.connect();
     fsck.setDisplayFullReport(); // i.e. -details
@@ -45,11 +52,13 @@ public class HbckTestingUtil {
     fsck.setFixHdfsOverlaps(fixHdfsOverlaps);
     fsck.setFixHdfsOrphans(fixHdfsOrphans);
     fsck.setFixVersionFile(fixVersionFile);
+    if (table != null) {
+      fsck.includeTable(table);
+    }
     fsck.onlineHbck();
     return fsck;
   }
 
-
   public static void assertNoErrors(HBaseFsck fsck) throws Exception {
     List<ERROR_CODE> errs = fsck.getErrors().getErrorList();
     assertEquals(new ArrayList<ERROR_CODE>(), errs);



Mime
View raw message