Return-Path: X-Original-To: apmail-hbase-commits-archive@www.apache.org Delivered-To: apmail-hbase-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id D2B2098BB for ; Thu, 12 Apr 2012 17:43:54 +0000 (UTC) Received: (qmail 24644 invoked by uid 500); 12 Apr 2012 17:43:54 -0000 Delivered-To: apmail-hbase-commits-archive@hbase.apache.org Received: (qmail 24616 invoked by uid 500); 12 Apr 2012 17:43:54 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 24609 invoked by uid 99); 12 Apr 2012 17:43:54 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 12 Apr 2012 17:43:54 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 12 Apr 2012 17:43:49 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 9273A238890B for ; Thu, 12 Apr 2012 17:43:27 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1325402 - in /hbase/branches/0.90: ./ src/main/java/org/apache/hadoop/hbase/util/ src/test/java/org/apache/hadoop/hbase/util/ Date: Thu, 12 Apr 2012 17:43:27 -0000 To: commits@hbase.apache.org From: jmhsieh@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120412174327.9273A238890B@eris.apache.org> Author: jmhsieh Date: Thu Apr 12 17:43:26 2012 New Revision: 1325402 URL: http://svn.apache.org/viewvc?rev=1325402&view=rev Log: HBASE-5719 Enhance hbck to sideline overlapped mega regions (Jimmy Xiang) Modified: hbase/branches/0.90/CHANGES.txt hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/util/RegionSplitCalculator.java hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitCalculator.java Modified: hbase/branches/0.90/CHANGES.txt URL: http://svn.apache.org/viewvc/hbase/branches/0.90/CHANGES.txt?rev=1325402&r1=1325401&r2=1325402&view=diff ============================================================================== --- hbase/branches/0.90/CHANGES.txt (original) +++ hbase/branches/0.90/CHANGES.txt Thu Apr 12 17:43:26 2012 @@ -24,6 +24,7 @@ Release 0.90.7 - Unreleased NEW FEATURE HBASE-5128 [uber hbck] Online automated repair of table integrity and region consistency problems HBASE-5599 [hbck] handle NO_VERSION_FILE and SHOULD_NOT_BE_DEPLOYED inconsistencies (fulin wang) + HBASE-5719 Enhance hbck to sideline overlapped mega regions (Jimmy Xiang) Release 0.90.6 - March 16, 2012 INCOMPATIBLE CHANGES Modified: hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java?rev=1325402&r1=1325401&r2=1325402&view=diff ============================================================================== --- hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java (original) +++ hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java Thu Apr 12 17:43:26 2012 @@ -22,8 +22,10 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Comparator; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.SortedSet; @@ -132,6 +134,7 @@ public class HBaseFsck { private static final int MAX_NUM_THREADS = 50; // #threads to contact regions private static final long THREADS_KEEP_ALIVE_SECONDS = 60; private static boolean rsSupportsOffline = true; + private static final int DEFAULT_OVERLAPS_TO_SIDELINE = 2; private static final int DEFAULT_MAX_MERGE = 5; /********************** @@ -143,7 +146,6 @@ public class HBaseFsck { private HBaseAdmin admin; private HTable meta; private ThreadPoolExecutor executor; // threads to retrieve data from regionservers - private int numThreads = MAX_NUM_THREADS; private long hbckStartMillis = System.currentTimeMillis(); /*********** @@ -161,6 +163,8 @@ public class HBaseFsck { // limit fixes to listed tables, if empty atttempt to fix all private List tablesToFix = new ArrayList(); private int maxMerge = DEFAULT_MAX_MERGE; // maximum number of overlapping regions to merge + private int maxOverlapsToSideline = DEFAULT_OVERLAPS_TO_SIDELINE; // maximum number of overlapping regions to sideline + private boolean sidelineBigOverlaps = false; // sideline overlaps with >maxMerge regions private boolean rerun = false; // if we tried to fix something, rerun hbck private static boolean summary = false; // if we want to print less output @@ -205,8 +209,8 @@ public class HBaseFsck { public HBaseFsck(Configuration conf) throws MasterNotRunningException, ZooKeeperConnectionException, IOException { this.conf = conf; - numThreads = conf.getInt("hbasefsck.numthreads", numThreads); - executor = new ThreadPoolExecutor(0, numThreads, + int numThreads = conf.getInt("hbasefsck.numthreads", MAX_NUM_THREADS); + executor = new ThreadPoolExecutor(1, numThreads, THREADS_KEEP_ALIVE_SECONDS, TimeUnit.SECONDS, new LinkedBlockingQueue()); } @@ -804,14 +808,14 @@ public class HBaseFsck { /** * Sideline a region dir (instead of deleting it) */ - void sidelineRegionDir(FileSystem fs, HbckInfo hi) + Path sidelineRegionDir(FileSystem fs, HbckInfo hi) throws IOException { String tableName = Bytes.toString(hi.getTableName()); Path regionDir = hi.getHdfsRegionDir(); if (!fs.exists(regionDir)) { LOG.warn("No previous " + regionDir + " exists. Continuing."); - return; + return null; } Path sidelineTableDir= new Path(getSidelineDir(), tableName); @@ -844,12 +848,15 @@ public class HBaseFsck { // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir, // it moves the src into the dst dir resulting in (foo/a/b). If // the dst does not exist, and the src a dir, src becomes dst. (foo/b) - for (FileStatus hfile : fs.listStatus(src)) { - success = fs.rename(hfile.getPath(), dst); - if (!success) { - String msg = "Unable to rename file " + src + " to " + dst; - LOG.error(msg); - throw new IOException(msg); + FileStatus[] hfiles = fs.listStatus(src); + if (hfiles != null && hfiles.length > 0) { + for (FileStatus hfile : hfiles) { + success = fs.rename(hfile.getPath(), dst); + if (!success) { + String msg = "Unable to rename file " + src + " to " + dst; + LOG.error(msg); + throw new IOException(msg); + } } } LOG.debug("Sideline directory contents:"); @@ -864,6 +871,7 @@ public class HBaseFsck { LOG.error(msg); throw new IOException(msg); } + return sidelineRegionDir; } /** @@ -1488,6 +1496,9 @@ public class HBaseFsck { // backwards regions final List backwards = new ArrayList(); + // sidelined big overlapped regions + final Map sidelinedRegions = new HashMap(); + // region split calculator final RegionSplitCalculator sc = new RegionSplitCalculator(cmp); @@ -1531,6 +1542,9 @@ public class HBaseFsck { private HTableDescriptor getHTD() { if (htds.size() == 1) { return (HTableDescriptor)htds.toArray()[0]; + } else { + LOG.error("None/Multiple table descriptors found for table '" + + tableName + "' regions: " + htds); } return null; } @@ -1711,13 +1725,21 @@ public class HBaseFsck { if (overlap.size() > maxMerge) { LOG.warn("Overlap group has " + overlap.size() + " overlapping " + - "regions which is greater than " + maxMerge + ", the max " + - "number of regions to merge."); + "regions which is greater than " + maxMerge + ", the max number of regions to merge"); + if (sidelineBigOverlaps) { + // we only sideline big overlapped groups that exceeds the max number of regions to merge + sidelineBigOverlaps(overlap); + } return; } + mergeOverlaps(overlap); + } + + void mergeOverlaps(Collection overlap) + throws IOException { LOG.info("== Merging regions into one region: " - + Joiner.on(",").join(overlap)); + + Joiner.on(",").join(overlap)); // get the min / max range and close all concerned regions Pair range = null; for (HbckInfo hi : overlap) { @@ -1781,7 +1803,48 @@ public class HBaseFsck { fixes++; } } - }; + + /** + * Sideline some regions in a big overlap group so that it + * will have fewer regions, and it is easier to merge them later on. + * + * @param bigOverlap the overlapped group with regions more than maxMerge + * @throws IOException + */ + void sidelineBigOverlaps( + Collection bigOverlap) throws IOException { + int overlapsToSideline = bigOverlap.size() - maxMerge; + if (overlapsToSideline > maxOverlapsToSideline) { + overlapsToSideline = maxOverlapsToSideline; + } + List regionsToSideline = + RegionSplitCalculator.findBigRanges(bigOverlap, overlapsToSideline); + FileSystem fs = FileSystem.get(conf); + for (HbckInfo regionToSideline: regionsToSideline) { + try { + LOG.info("Closing region: " + regionToSideline); + closeRegion(regionToSideline); + } catch (InterruptedException ie) { + LOG.warn("Was unable to close region " + regionToSideline.getRegionNameAsString() + + ". Interrupted."); + throw new IOException(ie); + } + + LOG.info("Offlining region: " + regionToSideline); + offline(regionToSideline.getRegionName()); + + LOG.info("Before sideline big overlapped region: " + regionToSideline.toString()); + Path sidelineRegionDir = sidelineRegionDir(fs, regionToSideline); + if (sidelineRegionDir != null) { + sidelinedRegions.put(sidelineRegionDir, regionToSideline); + LOG.info("After sidelined big overlapped region: " + + regionToSideline.getRegionNameAsString() + + " to " + sidelineRegionDir.toString()); + fixes++; + } + } + } + } /** * Check the region chain of this table. We are looking for holes, @@ -1866,16 +1929,22 @@ public class HBaseFsck { if (details) { // do full region split map dump - System.out.println("---- Table '" + this.tableName + System.out.println("---- Table '" + this.tableName + "': region split map"); dump(splits, regions); - System.out.println("---- Table '" + this.tableName + System.out.println("---- Table '" + this.tableName + "': overlap groups"); dumpOverlapProblems(overlapGroups); System.out.println("There are " + overlapGroups.keySet().size() + " overlap groups with " + overlapGroups.size() + " overlapping regions"); } + if (!sidelinedRegions.isEmpty()) { + LOG.warn("Sidelined big overlapped regions, please bulk load them!"); + System.out.println("---- Table '" + this.tableName + + "': sidelined big overlapped regions"); + dumpSidelinedRegions(sidelinedRegions); + } return errors.getErrorList().size() == originalErrorsCount; } @@ -1908,6 +1977,13 @@ public class HBaseFsck { } } + public void dumpSidelinedRegions(Map regions) { + for (Path k : regions.keySet()) { + System.out.println("To be bulk loaded sidelined region dir: " + + k.toString()); + } + } + public Multimap getOverlapGroups( String table) { TableInfo ti = tablesInfo.get(table); @@ -2736,6 +2812,14 @@ public class HBaseFsck { return fixVersionFile; } + public void setSidelineBigOverlaps(boolean sbo) { + this.sidelineBigOverlaps = sbo; + } + + public boolean shouldSidelineBigOverlaps() { + return sidelineBigOverlaps; + } + /** * @param mm maximum number of regions to merge into a single region. */ @@ -2747,6 +2831,14 @@ public class HBaseFsck { return maxMerge; } + public void setMaxOverlapsToSideline(int mo) { + this.maxOverlapsToSideline = mo; + } + + public int getMaxOverlapsToSideline() { + return maxOverlapsToSideline; + } + /** * Only fix tables specified by the list */ @@ -2796,8 +2888,10 @@ public class HBaseFsck { System.err.println(" -fixHdfsOverlaps Try to fix region overlaps in hdfs."); System.err.println(" -fixVersionFile Try to fix missing hbase.version file in hdfs."); System.err.println(" -maxMerge When fixing region overlaps, allow at most regions to merge. (n=" + DEFAULT_MAX_MERGE +" by default)"); + System.err.println(" -sidelineBigOverlaps When fixing region overlaps, allow to sideline big overlaps"); + System.err.println(" -maxOverlapsToSideline When fixing region overlaps, allow at most regions to sideline per group. (n=" + DEFAULT_OVERLAPS_TO_SIDELINE +" by default)"); System.err.println(""); - System.err.println(" -repair Shortcut for -fixAssignments -fixMeta -fixHdfsHoles -fixHdfsOrphans -fixHdfsOverlaps -fixVersionFile"); + System.err.println(" -repair Shortcut for -fixAssignments -fixMeta -fixHdfsHoles -fixHdfsOrphans -fixHdfsOverlaps -fixVersionFile -sidelineBigOverlaps"); System.err.println(" -repairHoles Shortcut for -fixAssignments -fixMeta -fixHdfsHoles -fixHdfsOrphans"); Runtime.getRuntime().exit(-2); @@ -2862,6 +2956,8 @@ public class HBaseFsck { fsck.setFixHdfsOverlaps(true); } else if (cmd.equals("-fixVersionFile")) { fsck.setFixVersionFile(true); + } else if (cmd.equals("-sidelineBigOverlaps")) { + fsck.setSidelineBigOverlaps(true); } else if (cmd.equals("-repair")) { // this attempts to merge overlapping hdfs regions, needs testing // under load @@ -2871,6 +2967,7 @@ public class HBaseFsck { fsck.setFixAssignments(true); fsck.setFixHdfsOverlaps(true); fsck.setFixVersionFile(true); + fsck.setSidelineBigOverlaps(true); } else if (cmd.equals("-repairHoles")) { // this will make all missing hdfs regions available but may lose data fsck.setFixHdfsHoles(true); @@ -2878,6 +2975,20 @@ public class HBaseFsck { fsck.setFixMeta(true); fsck.setFixAssignments(true); fsck.setFixHdfsOverlaps(false); + fsck.setSidelineBigOverlaps(false); + } else if (cmd.equals("-maxOverlapsToSideline")) { + if (i == args.length - 1) { + System.err.println("-maxOverlapsToSideline needs a numeric value argument."); + printUsageAndExit(); + } + try { + int maxOverlapsToSideline = Integer.parseInt(args[i+1]); + fsck.setMaxOverlapsToSideline(maxOverlapsToSideline); + } catch (NumberFormatException e) { + System.err.println("-maxOverlapsToSideline needs a numeric value argument."); + printUsageAndExit(); + } + i++; } else if (cmd.equals("-maxMerge")) { if (i == args.length - 1) { System.err.println("-maxMerge needs a numeric value argument."); Modified: hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/util/RegionSplitCalculator.java URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/util/RegionSplitCalculator.java?rev=1325402&r1=1325401&r2=1325402&view=diff ============================================================================== --- hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/util/RegionSplitCalculator.java (original) +++ hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/util/RegionSplitCalculator.java Thu Apr 12 17:43:26 2012 @@ -19,9 +19,12 @@ */ package org.apache.hadoop.hbase.util; +import java.util.ArrayList; import java.util.Collection; import java.util.Comparator; +import java.util.List; import java.util.Map.Entry; +import java.util.TreeMap; import java.util.TreeSet; import org.apache.commons.logging.Log; @@ -94,7 +97,7 @@ public class RegionSplitCalculator byte[] specialEndKey(R range) { byte[] end = range.getEndKey(); if (end.length == 0) { return ENDKEY; @@ -159,4 +162,75 @@ public class RegionSplitCalculator List + findBigRanges(Collection bigOverlap, int count) { + List bigRanges = new ArrayList(); + + // The key is the count of overlaps, + // The value is a list of ranges that have that many overlaps + TreeMap> overlapRangeMap = new TreeMap>(); + for (R r: bigOverlap) { + // Calculates the # of overlaps for each region + // and populates rangeOverlapMap + byte[] startKey = r.getStartKey(); + byte[] endKey = specialEndKey(r); + + int overlappedRegions = 0; + for (R rr: bigOverlap) { + byte[] start = rr.getStartKey(); + byte[] end = specialEndKey(rr); + + if (BYTES_COMPARATOR.compare(startKey, end) < 0 + && BYTES_COMPARATOR.compare(endKey, start) > 0) { + overlappedRegions++; + } + } + + // One region always overlaps with itself, + // so overlappedRegions should be more than 1 + // for actual overlaps. + if (overlappedRegions > 1) { + Integer key = Integer.valueOf(overlappedRegions); + List ranges = overlapRangeMap.get(key); + if (ranges == null) { + ranges = new ArrayList(); + overlapRangeMap.put(key, ranges); + } + ranges.add(r); + } + } + int toBeAdded = count; + for (Integer key: overlapRangeMap.descendingKeySet()) { + List chunk = overlapRangeMap.get(key); + int chunkSize = chunk.size(); + if (chunkSize <= toBeAdded) { + bigRanges.addAll(chunk); + toBeAdded -= chunkSize; + if (toBeAdded > 0) continue; + } else { + // Try to use the middle chunk in case the overlapping is + // chained, for example: [a, c), [b, e), [d, g), [f h)... + // In such a case, sideline the middle chunk will break + // the group efficiently. + int start = (chunkSize - toBeAdded)/2; + int end = start + toBeAdded; + for (int i = start; i < end; i++) { + bigRanges.add(chunk.get(i)); + } + } + break; + } + return bigRanges; + } } Modified: hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitCalculator.java URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitCalculator.java?rev=1325402&r1=1325401&r2=1325402&view=diff ============================================================================== --- hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitCalculator.java (original) +++ hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitCalculator.java Thu Apr 12 17:43:26 2012 @@ -20,9 +20,12 @@ package org.apache.hadoop.hbase.util; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import java.util.ArrayList; import java.util.Collection; import java.util.Comparator; +import java.util.List; import java.util.SortedSet; import org.apache.commons.logging.Log; @@ -347,4 +350,40 @@ public class TestRegionSplitCalculator { assertEquals(res, ":\t[, A]\t\n" + "A:\t[A, B]\t\n" + "B:\t[B, ]\t\n" + "null:\t\n"); } + + @Test + public void testBigRanges() { + SimpleRange ai = new SimpleRange(Bytes.toBytes("A"), Bytes.toBytes("I")); + SimpleRange ae = new SimpleRange(Bytes.toBytes("A"), Bytes.toBytes("E")); + SimpleRange ac = new SimpleRange(Bytes.toBytes("A"), Bytes.toBytes("C")); + + Collection bigOverlap = new ArrayList(); + bigOverlap.add(new SimpleRange(Bytes.toBytes("A"), Bytes.toBytes("E"))); + bigOverlap.add(new SimpleRange(Bytes.toBytes("A"), Bytes.toBytes("C"))); + bigOverlap.add(new SimpleRange(Bytes.toBytes("A"), Bytes.toBytes("B"))); + bigOverlap.add(new SimpleRange(Bytes.toBytes("B"), Bytes.toBytes("C"))); + bigOverlap.add(new SimpleRange(Bytes.toBytes("E"), Bytes.toBytes("H"))); + bigOverlap.add(ai); + bigOverlap.add(ae); + bigOverlap.add(ac); + + // Expect 1 range to be returned: ai + List bigRanges = RegionSplitCalculator.findBigRanges(bigOverlap, 1); + assertEquals(1, bigRanges.size()); + assertEquals(ai, bigRanges.get(0)); + + // Expect 3 ranges to be returned: ai, ae and ac + bigRanges = RegionSplitCalculator.findBigRanges(bigOverlap, 3); + assertEquals(3, bigRanges.size()); + assertEquals(ai, bigRanges.get(0)); + + SimpleRange r1 = bigRanges.get(1); + SimpleRange r2 = bigRanges.get(2); + assertEquals(Bytes.toString(r1.start), "A"); + assertEquals(Bytes.toString(r2.start), "A"); + String r1e = Bytes.toString(r1.end); + String r2e = Bytes.toString(r2.end); + assertTrue((r1e.equals("C") && r2e.equals("E")) + || (r1e.equals("E") && r2e.equals("C"))); + } }