hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jxi...@apache.org
Subject svn commit: r1443767 - in /hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util: HBaseFsck.java hbck/HFileCorruptionChecker.java
Date Thu, 07 Feb 2013 23:02:02 GMT
Author: jxiang
Date: Thu Feb  7 23:02:01 2013
New Revision: 1443767

URL: http://svn.apache.org/r1443767
Log:
HBASE-7776 Use ErrorReporter/Log instead of System.out in hbck

Modified:
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/HFileCorruptionChecker.java

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java?rev=1443767&r1=1443766&r2=1443767&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java Thu
Feb  7 23:02:01 2013
@@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.util;
 
 import java.io.IOException;
 import java.io.PrintWriter;
+import java.io.StringWriter;
 import java.net.URI;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -1416,8 +1417,9 @@ public class HBaseFsck extends Configure
         FSUtils.checkAccess(ugi, file, FsAction.WRITE);
       } catch (AccessControlException ace) {
         LOG.warn("Got AccessControlException when preCheckPermission ", ace);
-        System.err.println("Current user " + ugi.getUserName() + " does not have write perms
to " + file.getPath()
-            + ". Please rerun hbck as hdfs user " + file.getOwner());
+        errors.reportError(ERROR_CODE.WRONG_USAGE, "Current user " + ugi.getUserName()
+          + " does not have write perms to " + file.getPath()
+          + ". Please rerun hbck as hdfs user " + file.getOwner());
         throw new AccessControlException(ace);
       }
     }
@@ -2319,19 +2321,19 @@ public class HBaseFsck extends Configure
 
       if (details) {
         // do full region split map dump
-        System.out.println("---- Table '"  +  this.tableName
+        errors.print("---- Table '"  +  this.tableName
             + "': region split map");
         dump(splits, regions);
-        System.out.println("---- Table '"  +  this.tableName
+        errors.print("---- Table '"  +  this.tableName
             + "': overlap groups");
         dumpOverlapProblems(overlapGroups);
-        System.out.println("There are " + overlapGroups.keySet().size()
+        errors.print("There are " + overlapGroups.keySet().size()
             + " overlap groups with " + overlapGroups.size()
             + " overlapping regions");
       }
       if (!sidelinedRegions.isEmpty()) {
         LOG.warn("Sidelined big overlapped regions, please bulk load them!");
-        System.out.println("---- Table '"  +  this.tableName
+        errors.print("---- Table '"  +  this.tableName
             + "': sidelined big overlapped regions");
         dumpSidelinedRegions(sidelinedRegions);
       }
@@ -2346,13 +2348,15 @@ public class HBaseFsck extends Configure
      */
     void dump(SortedSet<byte[]> splits, Multimap<byte[], HbckInfo> regions) {
       // we display this way because the last end key should be displayed as well.
+      StringBuilder sb = new StringBuilder();
       for (byte[] k : splits) {
-        System.out.print(Bytes.toStringBinary(k) + ":\t");
+        sb.setLength(0); // clear out existing buffer, if any.
+        sb.append(Bytes.toStringBinary(k) + ":\t");
         for (HbckInfo r : regions.get(k)) {
-          System.out.print("[ "+ r.toString() + ", "
+          sb.append("[ "+ r.toString() + ", "
               + Bytes.toStringBinary(r.getEndKey())+ "]\t");
         }
-        System.out.println();
+        errors.print(sb.toString());
       }
     }
   }
@@ -2361,12 +2365,12 @@ public class HBaseFsck extends Configure
     // we display this way because the last end key should be displayed as
     // well.
     for (byte[] k : regions.keySet()) {
-      System.out.print(Bytes.toStringBinary(k) + ":\n");
+      errors.print(Bytes.toStringBinary(k) + ":");
       for (HbckInfo r : regions.get(k)) {
-        System.out.print("[ " + r.toString() + ", "
-            + Bytes.toStringBinary(r.getEndKey()) + "]\n");
+        errors.print("[ " + r.toString() + ", "
+            + Bytes.toStringBinary(r.getEndKey()) + "]");
       }
-      System.out.println("----");
+      errors.print("----");
     }
   }
 
@@ -2374,9 +2378,9 @@ public class HBaseFsck extends Configure
     for (Map.Entry<Path, HbckInfo> entry: regions.entrySet()) {
       String tableName = Bytes.toStringBinary(entry.getValue().getTableName());
       Path path = entry.getKey();
-      System.out.println("This sidelined region dir should be bulk loaded: "
+      errors.print("This sidelined region dir should be bulk loaded: "
         + path.toString());
-      System.out.println("Bulk load command looks like: "
+      errors.print("Bulk load command looks like: "
         + "hbase org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles "
         + path.toUri().getPath() + " "+ tableName);
     }
@@ -2807,23 +2811,25 @@ public class HBaseFsck extends Configure
    * Prints summary of all tables found on the system.
    */
   private void printTableSummary(SortedMap<String, TableInfo> tablesInfo) {
-    System.out.println("Summary:");
+    StringBuilder sb = new StringBuilder();
+    errors.print("Summary:");
     for (TableInfo tInfo : tablesInfo.values()) {
       if (errors.tableHasErrors(tInfo)) {
-        System.out.println("Table " + tInfo.getName() + " is inconsistent.");
+        errors.print("Table " + tInfo.getName() + " is inconsistent.");
       } else {
-        System.out.println("  " + tInfo.getName() + " is okay.");
+        errors.print("  " + tInfo.getName() + " is okay.");
       }
-      System.out.println("    Number of regions: " + tInfo.getNumRegions());
-      System.out.print("    Deployed on: ");
+      errors.print("    Number of regions: " + tInfo.getNumRegions());
+      sb.setLength(0); // clear out existing buffer, if any.
+      sb.append("    Deployed on: ");
       for (ServerName server : tInfo.deployedOn) {
-        System.out.print(" " + server.toString());
+        sb.append(" " + server.toString());
       }
-      System.out.println();
+      errors.print(sb.toString());
     }
   }
 
-  private static ErrorReporter getErrorReporter(
+  static ErrorReporter getErrorReporter(
       final Configuration conf) throws ClassNotFoundException {
     Class<? extends ErrorReporter> reporter = conf.getClass("hbasefsck.errorreporter",
PrintingErrorReporter.class, ErrorReporter.class);
     return (ErrorReporter)ReflectionUtils.newInstance(reporter, conf);
@@ -2836,7 +2842,8 @@ public class HBaseFsck extends Configure
       MULTI_DEPLOYED, SHOULD_NOT_BE_DEPLOYED, MULTI_META_REGION, RS_CONNECT_FAILURE,
       FIRST_REGION_STARTKEY_NOT_EMPTY, LAST_REGION_ENDKEY_NOT_EMPTY, DUPE_STARTKEYS,
       HOLE_IN_REGION_CHAIN, OVERLAP_IN_REGION_CHAIN, REGION_CYCLE, DEGENERATE_REGION,
-      ORPHAN_HDFS_REGION, LINGERING_SPLIT_PARENT, NO_TABLEINFO_FILE, LINGERING_REFERENCE_HFILE
+      ORPHAN_HDFS_REGION, LINGERING_SPLIT_PARENT, NO_TABLEINFO_FILE, LINGERING_REFERENCE_HFILE,
+      WRONG_USAGE
     }
     public void clear();
     public void report(String message);
@@ -2870,6 +2877,11 @@ public class HBaseFsck extends Configure
     }
 
     public synchronized void reportError(ERROR_CODE errorCode, String message) {
+      if (errorCode == ERROR_CODE.WRONG_USAGE) {
+        System.err.println(message);
+        return;
+      }
+
       errorList.add(errorCode);
       if (!summary) {
         System.out.println("ERROR: " + message);
@@ -3358,48 +3370,53 @@ public class HBaseFsck extends Configure
   }
 
   protected HBaseFsck printUsageAndExit() {
-    System.err.println("Usage: fsck [opts] {only tables}");
-    System.err.println(" where [opts] are:");
-    System.err.println("   -help Display help options (this)");
-    System.err.println("   -details Display full report of all regions.");
-    System.err.println("   -timelag <timeInSeconds>  Process only regions that " +
+    StringWriter sw = new StringWriter(2048);
+    PrintWriter out = new PrintWriter(sw);
+    out.println("Usage: fsck [opts] {only tables}");
+    out.println(" where [opts] are:");
+    out.println("   -help Display help options (this)");
+    out.println("   -details Display full report of all regions.");
+    out.println("   -timelag <timeInSeconds>  Process only regions that " +
                        " have not experienced any metadata updates in the last " +
                        " <timeInSeconds> seconds.");
-    System.err.println("   -sleepBeforeRerun <timeInSeconds> Sleep this many seconds"
+
+    out.println("   -sleepBeforeRerun <timeInSeconds> Sleep this many seconds" +
         " before checking if the fix worked if run with -fix");
-    System.err.println("   -summary Print only summary of the tables and status.");
-    System.err.println("   -metaonly Only check the state of ROOT and META tables.");
-    System.err.println("   -sidelineDir <hdfs://> HDFS path to backup existing meta
and root.");
-
-    System.err.println("");
-    System.err.println("  Metadata Repair options: (expert features, use with caution!)");
-    System.err.println("   -fix              Try to fix region assignments.  This is for
backwards compatiblity");
-    System.err.println("   -fixAssignments   Try to fix region assignments.  Replaces the
old -fix");
-    System.err.println("   -fixMeta          Try to fix meta problems.  This assumes HDFS
region info is good.");
-    System.err.println("   -noHdfsChecking   Don't load/check region info from HDFS."
+    out.println("   -summary Print only summary of the tables and status.");
+    out.println("   -metaonly Only check the state of ROOT and META tables.");
+    out.println("   -sidelineDir <hdfs://> HDFS path to backup existing meta and root.");
+
+    out.println("");
+    out.println("  Metadata Repair options: (expert features, use with caution!)");
+    out.println("   -fix              Try to fix region assignments.  This is for backwards
compatiblity");
+    out.println("   -fixAssignments   Try to fix region assignments.  Replaces the old -fix");
+    out.println("   -fixMeta          Try to fix meta problems.  This assumes HDFS region
info is good.");
+    out.println("   -noHdfsChecking   Don't load/check region info from HDFS."
         + " Assumes META region info is good. Won't check/fix any HDFS issue, e.g. hole,
orphan, or overlap");
-    System.err.println("   -fixHdfsHoles     Try to fix region holes in hdfs.");
-    System.err.println("   -fixHdfsOrphans   Try to fix region dirs with no .regioninfo file
in hdfs");
-    System.err.println("   -fixTableOrphans  Try to fix table dirs with no .tableinfo file
in hdfs (online mode only)");
-    System.err.println("   -fixHdfsOverlaps  Try to fix region overlaps in hdfs.");
-    System.err.println("   -fixVersionFile   Try to fix missing hbase.version file in hdfs.");
-    System.err.println("   -maxMerge <n>     When fixing region overlaps, allow at
most <n> regions to merge. (n=" + DEFAULT_MAX_MERGE +" by default)");
-    System.err.println("   -sidelineBigOverlaps  When fixing region overlaps, allow to sideline
big overlaps");
-    System.err.println("   -maxOverlapsToSideline <n>  When fixing region overlaps,
allow at most <n> regions to sideline per group. (n=" + DEFAULT_OVERLAPS_TO_SIDELINE
+" by default)");
-    System.err.println("   -fixSplitParents  Try to force offline split parents to be online.");
-    System.err.println("   -ignorePreCheckPermission  ignore filesystem permission pre-check");
-    System.err.println("   -fixReferenceFiles  Try to offline lingering reference store files");
-
-    System.err.println("");
-    System.err.println("  Datafile Repair options: (expert features, use with caution!)");
-    System.err.println("   -checkCorruptHFiles     Check all Hfiles by opening them to make
sure they are valid");
-    System.err.println("   -sidelineCorruptHfiles  Quarantine corrupted HFiles.  implies
-checkCorruptHfiles");
-
-    System.err.println("");
-    System.err.println("  Metadata Repair shortcuts");
-    System.err.println("   -repair           Shortcut for -fixAssignments -fixMeta -fixHdfsHoles
" +
+    out.println("   -fixHdfsHoles     Try to fix region holes in hdfs.");
+    out.println("   -fixHdfsOrphans   Try to fix region dirs with no .regioninfo file in
hdfs");
+    out.println("   -fixTableOrphans  Try to fix table dirs with no .tableinfo file in hdfs
(online mode only)");
+    out.println("   -fixHdfsOverlaps  Try to fix region overlaps in hdfs.");
+    out.println("   -fixVersionFile   Try to fix missing hbase.version file in hdfs.");
+    out.println("   -maxMerge <n>     When fixing region overlaps, allow at most <n>
regions to merge. (n=" + DEFAULT_MAX_MERGE +" by default)");
+    out.println("   -sidelineBigOverlaps  When fixing region overlaps, allow to sideline
big overlaps");
+    out.println("   -maxOverlapsToSideline <n>  When fixing region overlaps, allow
at most <n> regions to sideline per group. (n=" + DEFAULT_OVERLAPS_TO_SIDELINE +" by
default)");
+    out.println("   -fixSplitParents  Try to force offline split parents to be online.");
+    out.println("   -ignorePreCheckPermission  ignore filesystem permission pre-check");
+    out.println("   -fixReferenceFiles  Try to offline lingering reference store files");
+
+    out.println("");
+    out.println("  Datafile Repair options: (expert features, use with caution!)");
+    out.println("   -checkCorruptHFiles     Check all Hfiles by opening them to make sure
they are valid");
+    out.println("   -sidelineCorruptHfiles  Quarantine corrupted HFiles.  implies -checkCorruptHfiles");
+
+    out.println("");
+    out.println("  Metadata Repair shortcuts");
+    out.println("   -repair           Shortcut for -fixAssignments -fixMeta -fixHdfsHoles
" +
         "-fixHdfsOrphans -fixHdfsOverlaps -fixVersionFile -sidelineBigOverlaps -fixReferenceFiles");
-    System.err.println("   -repairHoles      Shortcut for -fixAssignments -fixMeta -fixHdfsHoles");
+    out.println("   -repairHoles      Shortcut for -fixAssignments -fixMeta -fixHdfsHoles");
+
+    out.flush();
+    errors.reportError(ERROR_CODE.WRONG_USAGE, sw.toString());
 
     setRetCode(-2);
     return this;
@@ -3445,39 +3462,40 @@ public class HBaseFsck extends Configure
         setDisplayFullReport();
       } else if (cmd.equals("-timelag")) {
         if (i == args.length - 1) {
-          System.err.println("HBaseFsck: -timelag needs a value.");
+          errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -timelag needs a value.");
           return printUsageAndExit();
         }
         try {
           long timelag = Long.parseLong(args[i+1]);
           setTimeLag(timelag);
         } catch (NumberFormatException e) {
-          System.err.println("-timelag needs a numeric value.");
+          errors.reportError(ERROR_CODE.WRONG_USAGE, "-timelag needs a numeric value.");
           return printUsageAndExit();
         }
         i++;
       } else if (cmd.equals("-sleepBeforeRerun")) {
         if (i == args.length - 1) {
-          System.err.println("HBaseFsck: -sleepBeforeRerun needs a value.");
+          errors.reportError(ERROR_CODE.WRONG_USAGE,
+            "HBaseFsck: -sleepBeforeRerun needs a value.");
           return printUsageAndExit();
         }
         try {
           sleepBeforeRerun = Long.parseLong(args[i+1]);
         } catch (NumberFormatException e) {
-          System.err.println("-sleepBeforeRerun needs a numeric value.");
+          errors.reportError(ERROR_CODE.WRONG_USAGE, "-sleepBeforeRerun needs a numeric value.");
           return printUsageAndExit();
         }
         i++;
       } else if (cmd.equals("-sidelineDir")) {
         if (i == args.length - 1) {
-          System.err.println("HBaseFsck: -sidelineDir needs a value.");
+          errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -sidelineDir needs a value.");
           return printUsageAndExit();
         }
         i++;
         setSidelineDir(args[i]);
       } else if (cmd.equals("-fix")) {
-        System.err.println("This option is deprecated, please use " +
-          "-fixAssignments instead.");
+        errors.reportError(ERROR_CODE.WRONG_USAGE,
+          "This option is deprecated, please use  -fixAssignments instead.");
         setFixAssignments(true);
       } else if (cmd.equals("-fixAssignments")) {
         setFixAssignments(true);
@@ -3532,27 +3550,31 @@ public class HBaseFsck extends Configure
         setCheckHdfs(true);
       } else if (cmd.equals("-maxOverlapsToSideline")) {
         if (i == args.length - 1) {
-          System.err.println("-maxOverlapsToSideline needs a numeric value argument.");
+          errors.reportError(ERROR_CODE.WRONG_USAGE,
+            "-maxOverlapsToSideline needs a numeric value argument.");
           return printUsageAndExit();
         }
         try {
           int maxOverlapsToSideline = Integer.parseInt(args[i+1]);
           setMaxOverlapsToSideline(maxOverlapsToSideline);
         } catch (NumberFormatException e) {
-          System.err.println("-maxOverlapsToSideline needs a numeric value argument.");
+          errors.reportError(ERROR_CODE.WRONG_USAGE,
+            "-maxOverlapsToSideline needs a numeric value argument.");
           return printUsageAndExit();
         }
         i++;
       } else if (cmd.equals("-maxMerge")) {
         if (i == args.length - 1) {
-          System.err.println("-maxMerge needs a numeric value argument.");
+          errors.reportError(ERROR_CODE.WRONG_USAGE,
+            "-maxMerge needs a numeric value argument.");
           return printUsageAndExit();
         }
         try {
           int maxMerge = Integer.parseInt(args[i+1]);
           setMaxMerge(maxMerge);
         } catch (NumberFormatException e) {
-          System.err.println("-maxMerge needs a numeric value argument.");
+          errors.reportError(ERROR_CODE.WRONG_USAGE,
+            "-maxMerge needs a numeric value argument.");
           return printUsageAndExit();
         }
         i++;
@@ -3561,11 +3583,11 @@ public class HBaseFsck extends Configure
       } else if (cmd.equals("-metaonly")) {
         setCheckMetaOnly();
       } else if (cmd.startsWith("-")) {
-        System.err.println("Unrecognized option:" + cmd);
+        errors.reportError(ERROR_CODE.WRONG_USAGE, "Unrecognized option:" + cmd);
         return printUsageAndExit();
       } else {
         includeTable(cmd);
-        System.out.println("Allow checking/fixes for table: " + cmd);
+        errors.print("Allow checking/fixes for table: " + cmd);
       }
     }
 
@@ -3597,9 +3619,7 @@ public class HBaseFsck extends Configure
         tableDirs = FSUtils.getTableDirs(FSUtils.getCurrentFileSystem(getConf()), rootdir);
       }
       hfcc.checkTables(tableDirs);
-      PrintWriter out = new PrintWriter(System.out);
-      hfcc.report(out);
-      out.flush();
+      hfcc.report(errors);
     }
 
     // check and fix table integrity, region consistency.
@@ -3634,13 +3654,22 @@ public class HBaseFsck extends Configure
    * ls -r for debugging purposes
    */
   void debugLsr(Path p) throws IOException {
-    debugLsr(getConf(), p);
+    debugLsr(getConf(), p, errors);
+  }
+
+  /**
+   * ls -r for debugging purposes
+   */
+  public static void debugLsr(Configuration conf,
+      Path p) throws IOException {
+    debugLsr(conf, p, new PrintingErrorReporter());
   }
 
   /**
    * ls -r for debugging purposes
    */
-  public static void debugLsr(Configuration conf, Path p) throws IOException {
+  public static void debugLsr(Configuration conf,
+      Path p, ErrorReporter errors) throws IOException {
     if (!LOG.isDebugEnabled() || p == null) {
       return;
     }
@@ -3650,7 +3679,7 @@ public class HBaseFsck extends Configure
       // nothing
       return;
     }
-    System.out.println(p);
+    errors.print(p.toString());
 
     if (fs.isFile(p)) {
       return;
@@ -3659,7 +3688,7 @@ public class HBaseFsck extends Configure
     if (fs.getFileStatus(p).isDir()) {
       FileStatus[] fss= fs.listStatus(p);
       for (FileStatus status : fss) {
-        debugLsr(conf, status.getPath());
+        debugLsr(conf, status.getPath(), errors);
       }
     }
   }

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/HFileCorruptionChecker.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/HFileCorruptionChecker.java?rev=1443767&r1=1443766&r2=1443767&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/HFileCorruptionChecker.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/HFileCorruptionChecker.java
Thu Feb  7 23:02:01 2013
@@ -19,7 +19,6 @@ package org.apache.hadoop.hbase.util.hbc
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
-import java.io.PrintWriter;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashSet;
@@ -46,7 +45,7 @@ import org.apache.hadoop.hbase.io.hfile.
 import org.apache.hadoop.hbase.util.FSUtils.FamilyDirFilter;
 import org.apache.hadoop.hbase.util.FSUtils.HFileFilter;
 import org.apache.hadoop.hbase.util.FSUtils.RegionDirFilter;
-import org.apache.hadoop.hbase.util.HBaseFsck;
+import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter;
 
 /**
  * This class marches through all of the region's hfiles and verifies that
@@ -341,22 +340,22 @@ public class HFileCorruptionChecker {
    * Print a human readable summary of hfile quarantining operations.
    * @param out
    */
-  public void report(PrintWriter out) {
-    out.println("Checked " + hfilesChecked.get() + " hfile for corruption");
-    out.println("  HFiles corrupted:                  " + corrupted.size());
+  public void report(ErrorReporter out) {
+    out.print("Checked " + hfilesChecked.get() + " hfile for corruption");
+    out.print("  HFiles corrupted:                  " + corrupted.size());
     if (inQuarantineMode) {
-      out.println("    HFiles successfully quarantined: " + quarantined.size());
+      out.print("    HFiles successfully quarantined: " + quarantined.size());
       for (Path sq : quarantined) {
-        out.println("      " + sq);
+        out.print("      " + sq);
       }
-      out.println("    HFiles failed quarantine:        " + failures.size());
+      out.print("    HFiles failed quarantine:        " + failures.size());
       for (Path fq : failures) {
-        out.println("      " + fq);
+        out.print("      " + fq);
       }
     }
-    out.println("    HFiles moved while checking:     " + missing.size());
+    out.print("    HFiles moved while checking:     " + missing.size());
     for (Path mq : missing) {
-      out.println("      " + mq);
+      out.print("      " + mq);
     }
 
     String initialState = (corrupted.size() == 0) ? "OK" : "CORRUPTED";
@@ -364,9 +363,9 @@ public class HFileCorruptionChecker {
         : "CORRUPTED";
 
     if (inQuarantineMode) {
-      out.println("Summary: " + initialState + " => " + fixedState);
+      out.print("Summary: " + initialState + " => " + fixedState);
     } else {
-      out.println("Summary: " + initialState);
+      out.print("Summary: " + initialState);
     }
   }
 }



Mime
View raw message