Return-Path: X-Original-To: apmail-hbase-commits-archive@www.apache.org Delivered-To: apmail-hbase-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 7AAA518DD5 for ; Mon, 2 Nov 2015 03:51:29 +0000 (UTC) Received: (qmail 13837 invoked by uid 500); 2 Nov 2015 03:51:29 -0000 Delivered-To: apmail-hbase-commits-archive@hbase.apache.org Received: (qmail 13803 invoked by uid 500); 2 Nov 2015 03:51:29 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 13790 invoked by uid 99); 2 Nov 2015 03:51:29 -0000 Received: from Unknown (HELO spamd1-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 02 Nov 2015 03:51:29 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd1-us-west.apache.org (ASF Mail Server at spamd1-us-west.apache.org) with ESMTP id BF532C22E6 for ; Mon, 2 Nov 2015 03:51:28 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd1-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: 0.99 X-Spam-Level: X-Spam-Status: No, score=0.99 tagged_above=-999 required=6.31 tests=[KAM_LAZY_DOMAIN_SECURITY=1, T_RP_MATCHES_RCVD=-0.01] autolearn=disabled Received: from mx1-us-west.apache.org ([10.40.0.8]) by localhost (spamd1-us-west.apache.org [10.40.0.7]) (amavisd-new, port 10024) with ESMTP id UoKDkv7p_NBb for ; Mon, 2 Nov 2015 03:51:24 +0000 (UTC) Received: from mailrelay1-us-west.apache.org (mailrelay1-us-west.apache.org [209.188.14.139]) by mx1-us-west.apache.org (ASF Mail Server at mx1-us-west.apache.org) with ESMTP id 0EF692385F for ; Mon, 2 Nov 2015 03:51:12 +0000 (UTC) Received: from svn01-us-west.apache.org (svn.apache.org [10.41.0.6]) by mailrelay1-us-west.apache.org (ASF Mail Server at mailrelay1-us-west.apache.org) with ESMTP id B8C60E1022 for ; Mon, 2 Nov 2015 03:51:10 +0000 (UTC) Received: from svn01-us-west.apache.org (localhost [127.0.0.1]) by svn01-us-west.apache.org (ASF Mail Server at svn01-us-west.apache.org) with ESMTP id B7B813A078D for ; Mon, 2 Nov 2015 03:51:10 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1711891 [45/50] - in /hbase/hbase.apache.org/trunk: ./ apidocs/ apidocs/org/apache/hadoop/hbase/ apidocs/org/apache/hadoop/hbase/class-use/ apidocs/org/apache/hadoop/hbase/client/ apidocs/org/apache/hadoop/hbase/client/class-use/ apidocs/o... Date: Mon, 02 Nov 2015 03:51:05 -0000 To: commits@hbase.apache.org From: misty@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20151102035110.B7B813A078D@svn01-us-west.apache.org> Modified: hbase/hbase.apache.org/trunk/xref/org/apache/hadoop/hbase/master/CatalogJanitor.html URL: http://svn.apache.org/viewvc/hbase/hbase.apache.org/trunk/xref/org/apache/hadoop/hbase/master/CatalogJanitor.html?rev=1711891&r1=1711890&r2=1711891&view=diff ============================================================================== --- hbase/hbase.apache.org/trunk/xref/org/apache/hadoop/hbase/master/CatalogJanitor.html (original) +++ hbase/hbase.apache.org/trunk/xref/org/apache/hadoop/hbase/master/CatalogJanitor.html Mon Nov 2 03:51:02 2015 @@ -123,8 +123,8 @@ 113 * parent regioninfos 114 * @throws IOException 115 */ -116 Triple<Integer, Map<HRegionInfo, Result>, Map<HRegionInfo, Result>> getMergedRegionsAndSplitParents() -117 throws IOException { +116 Triple<Integer, Map<HRegionInfo, Result>, Map<HRegionInfo, Result>> +117 getMergedRegionsAndSplitParents() throws IOException { 118 return getMergedRegionsAndSplitParents(null); 119 } 120 @@ -138,8 +138,8 @@ 128 * parent regioninfos 129 * @throws IOException 130 */ -131 Triple<Integer, Map<HRegionInfo, Result>, Map<HRegionInfo, Result>> getMergedRegionsAndSplitParents( -132 final TableName tableName) throws IOException { +131 Triple<Integer, Map<HRegionInfo, Result>, Map<HRegionInfo, Result>> +132 getMergedRegionsAndSplitParents(final TableName tableName) throws IOException { 133 final boolean isTableSpecified = (tableName != null); 134 // TODO: Only works with single hbase:meta region currently. Fix. 135 final AtomicInteger count = new AtomicInteger(0); @@ -155,7 +155,7 @@ 145 public boolean visit(Result r) throws IOException { 146 if (r == null || r.isEmpty()) return true; 147 count.incrementAndGet(); -148 HRegionInfo info = HRegionInfo.getHRegionInfo(r); +148 HRegionInfo info = MetaTableAccessor.getHRegionInfo(r); 149 if (info == null) return true; // Keep scanning 150 if (isTableSpecified 151 && info.getTable().compareTo(tableName) > 0) { @@ -235,215 +235,216 @@ 225 int mergeCleaned = 0; 226 Map<HRegionInfo, Result> mergedRegions = scanTriple.getSecond(); 227 for (Map.Entry<HRegionInfo, Result> e : mergedRegions.entrySet()) { -228 HRegionInfo regionA = HRegionInfo.getHRegionInfo(e.getValue(), -229 HConstants.MERGEA_QUALIFIER); -230 HRegionInfo regionB = HRegionInfo.getHRegionInfo(e.getValue(), -231 HConstants.MERGEB_QUALIFIER); -232 if (regionA == null || regionB == null) { -233 LOG.warn("Unexpected references regionA=" -234 + (regionA == null ? "null" : regionA.getRegionNameAsString()) -235 + ",regionB=" -236 + (regionB == null ? "null" : regionB.getRegionNameAsString()) -237 + " in merged region " + e.getKey().getRegionNameAsString()); -238 } else { -239 if (cleanMergeRegion(e.getKey(), regionA, regionB)) { -240 mergeCleaned++; -241 } -242 } -243 } -244 /** -245 * clean split parents -246 */ -247 Map<HRegionInfo, Result> splitParents = scanTriple.getThird(); -248 -249 // Now work on our list of found parents. See if any we can clean up. -250 int splitCleaned = 0; -251 // regions whose parents are still around -252 HashSet<String> parentNotCleaned = new HashSet<String>(); -253 for (Map.Entry<HRegionInfo, Result> e : splitParents.entrySet()) { -254 if (!parentNotCleaned.contains(e.getKey().getEncodedName()) && -255 cleanParent(e.getKey(), e.getValue())) { -256 splitCleaned++; -257 } else { -258 // We could not clean the parent, so it's daughters should not be cleaned either (HBASE-6160) -259 PairOfSameType<HRegionInfo> daughters = HRegionInfo.getDaughterRegions(e.getValue()); -260 parentNotCleaned.add(daughters.getFirst().getEncodedName()); -261 parentNotCleaned.add(daughters.getSecond().getEncodedName()); -262 } -263 } -264 if ((mergeCleaned + splitCleaned) != 0) { -265 LOG.info("Scanned " + count + " catalog row(s), gc'd " + mergeCleaned -266 + " unreferenced merged region(s) and " + splitCleaned -267 + " unreferenced parent region(s)"); -268 } else if (LOG.isTraceEnabled()) { -269 LOG.trace("Scanned " + count + " catalog row(s), gc'd " + mergeCleaned -270 + " unreferenced merged region(s) and " + splitCleaned -271 + " unreferenced parent region(s)"); -272 } -273 return mergeCleaned + splitCleaned; -274 } finally { -275 alreadyRunning.set(false); -276 } -277 } -278 -279 /** -280 * Compare HRegionInfos in a way that has split parents sort BEFORE their -281 * daughters. -282 */ -283 static class SplitParentFirstComparator implements Comparator<HRegionInfo> { -284 Comparator<byte[]> rowEndKeyComparator = new Bytes.RowEndKeyComparator(); -285 @Override -286 public int compare(HRegionInfo left, HRegionInfo right) { -287 // This comparator differs from the one HRegionInfo in that it sorts -288 // parent before daughters. -289 if (left == null) return -1; -290 if (right == null) return 1; -291 // Same table name. -292 int result = left.getTable().compareTo(right.getTable()); -293 if (result != 0) return result; -294 // Compare start keys. -295 result = Bytes.compareTo(left.getStartKey(), right.getStartKey()); -296 if (result != 0) return result; -297 // Compare end keys, but flip the operands so parent comes first -298 result = rowEndKeyComparator.compare(right.getEndKey(), left.getEndKey()); -299 -300 return result; -301 } -302 } -303 -304 /** -305 * If daughters no longer hold reference to the parents, delete the parent. -306 * @param parent HRegionInfo of split offlined parent -307 * @param rowContent Content of <code>parent</code> row in -308 * <code>metaRegionName</code> -309 * @return True if we removed <code>parent</code> from meta table and from -310 * the filesystem. -311 * @throws IOException -312 */ -313 boolean cleanParent(final HRegionInfo parent, Result rowContent) -314 throws IOException { -315 boolean result = false; -316 // Check whether it is a merged region and not clean reference -317 // No necessary to check MERGEB_QUALIFIER because these two qualifiers will -318 // be inserted/deleted together -319 if (rowContent.getValue(HConstants.CATALOG_FAMILY, -320 HConstants.MERGEA_QUALIFIER) != null) { -321 // wait cleaning merge region first -322 return result; -323 } -324 // Run checks on each daughter split. -325 PairOfSameType<HRegionInfo> daughters = HRegionInfo.getDaughterRegions(rowContent); -326 Pair<Boolean, Boolean> a = checkDaughterInFs(parent, daughters.getFirst()); -327 Pair<Boolean, Boolean> b = checkDaughterInFs(parent, daughters.getSecond()); -328 if (hasNoReferences(a) && hasNoReferences(b)) { -329 LOG.debug("Deleting region " + parent.getRegionNameAsString() + -330 " because daughter splits no longer hold references"); -331 FileSystem fs = this.services.getMasterFileSystem().getFileSystem(); -332 if (LOG.isTraceEnabled()) LOG.trace("Archiving parent region: " + parent); -333 HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, parent); -334 MetaTableAccessor.deleteRegion(this.connection, parent); -335 result = true; -336 } -337 return result; -338 } -339 -340 /** -341 * @param p A pair where the first boolean says whether or not the daughter -342 * region directory exists in the filesystem and then the second boolean says -343 * whether the daughter has references to the parent. -344 * @return True the passed <code>p</code> signifies no references. -345 */ -346 private boolean hasNoReferences(final Pair<Boolean, Boolean> p) { -347 return !p.getFirst() || !p.getSecond(); -348 } -349 -350 /** -351 * Checks if a daughter region -- either splitA or splitB -- still holds -352 * references to parent. -353 * @param parent Parent region -354 * @param daughter Daughter region -355 * @return A pair where the first boolean says whether or not the daughter -356 * region directory exists in the filesystem and then the second boolean says -357 * whether the daughter has references to the parent. -358 * @throws IOException -359 */ -360 Pair<Boolean, Boolean> checkDaughterInFs(final HRegionInfo parent, final HRegionInfo daughter) -361 throws IOException { -362 if (daughter == null) { -363 return new Pair<Boolean, Boolean>(Boolean.FALSE, Boolean.FALSE); -364 } -365 -366 FileSystem fs = this.services.getMasterFileSystem().getFileSystem(); -367 Path rootdir = this.services.getMasterFileSystem().getRootDir(); -368 Path tabledir = FSUtils.getTableDir(rootdir, daughter.getTable()); -369 -370 Path daughterRegionDir = new Path(tabledir, daughter.getEncodedName()); -371 -372 HRegionFileSystem regionFs = null; -373 -374 try { -375 if (!FSUtils.isExists(fs, daughterRegionDir)) { -376 return new Pair<Boolean, Boolean>(Boolean.FALSE, Boolean.FALSE); -377 } -378 } catch (IOException ioe) { -379 LOG.warn("Error trying to determine if daughter region exists, " + -380 "assuming exists and has references", ioe); -381 return new Pair<Boolean, Boolean>(Boolean.TRUE, Boolean.TRUE); -382 } -383 -384 try { -385 regionFs = HRegionFileSystem.openRegionFromFileSystem( -386 this.services.getConfiguration(), fs, tabledir, daughter, true); -387 } catch (IOException e) { -388 LOG.warn("Error trying to determine referenced files from : " + daughter.getEncodedName() -389 + ", to: " + parent.getEncodedName() + " assuming has references", e); -390 return new Pair<Boolean, Boolean>(Boolean.TRUE, Boolean.TRUE); -391 } -392 -393 boolean references = false; -394 HTableDescriptor parentDescriptor = getTableDescriptor(parent.getTable()); -395 for (HColumnDescriptor family: parentDescriptor.getFamilies()) { -396 if ((references = regionFs.hasReferences(family.getNameAsString()))) { -397 break; -398 } -399 } -400 return new Pair<Boolean, Boolean>(Boolean.TRUE, Boolean.valueOf(references)); -401 } -402 -403 private HTableDescriptor getTableDescriptor(final TableName tableName) -404 throws FileNotFoundException, IOException { -405 return this.services.getTableDescriptors().get(tableName); -406 } -407 -408 /** -409 * Checks if the specified region has merge qualifiers, if so, try to clean -410 * them -411 * @param region -412 * @return true if the specified region doesn't have merge qualifier now -413 * @throws IOException -414 */ -415 public boolean cleanMergeQualifier(final HRegionInfo region) -416 throws IOException { -417 // Get merge regions if it is a merged region and already has merge -418 // qualifier -419 Pair<HRegionInfo, HRegionInfo> mergeRegions = MetaTableAccessor -420 .getRegionsFromMergeQualifier(this.services.getConnection(), -421 region.getRegionName()); -422 if (mergeRegions == null -423 || (mergeRegions.getFirst() == null && mergeRegions.getSecond() == null)) { -424 // It doesn't have merge qualifier, no need to clean -425 return true; -426 } -427 // It shouldn't happen, we must insert/delete these two qualifiers together -428 if (mergeRegions.getFirst() == null || mergeRegions.getSecond() == null) { -429 LOG.error("Merged region " + region.getRegionNameAsString() -430 + " has only one merge qualifier in META."); -431 return false; -432 } -433 return cleanMergeRegion(region, mergeRegions.getFirst(), -434 mergeRegions.getSecond()); -435 } -436 } +228 PairOfSameType<HRegionInfo> p = MetaTableAccessor.getMergeRegions(e.getValue()); +229 HRegionInfo regionA = p.getFirst(); +230 HRegionInfo regionB = p.getSecond(); +231 if (regionA == null || regionB == null) { +232 LOG.warn("Unexpected references regionA=" +233 + (regionA == null ? "null" : regionA.getRegionNameAsString()) +234 + ",regionB=" +235 + (regionB == null ? "null" : regionB.getRegionNameAsString()) +236 + " in merged region " + e.getKey().getRegionNameAsString()); +237 } else { +238 if (cleanMergeRegion(e.getKey(), regionA, regionB)) { +239 mergeCleaned++; +240 } +241 } +242 } +243 /** +244 * clean split parents +245 */ +246 Map<HRegionInfo, Result> splitParents = scanTriple.getThird(); +247 +248 // Now work on our list of found parents. See if any we can clean up. +249 int splitCleaned = 0; +250 // regions whose parents are still around +251 HashSet<String> parentNotCleaned = new HashSet<String>(); +252 for (Map.Entry<HRegionInfo, Result> e : splitParents.entrySet()) { +253 if (!parentNotCleaned.contains(e.getKey().getEncodedName()) && +254 cleanParent(e.getKey(), e.getValue())) { +255 splitCleaned++; +256 } else { +257 // We could not clean the parent, so it's daughters should not be +258 // cleaned either (HBASE-6160) +259 PairOfSameType<HRegionInfo> daughters = +260 MetaTableAccessor.getDaughterRegions(e.getValue()); +261 parentNotCleaned.add(daughters.getFirst().getEncodedName()); +262 parentNotCleaned.add(daughters.getSecond().getEncodedName()); +263 } +264 } +265 if ((mergeCleaned + splitCleaned) != 0) { +266 LOG.info("Scanned " + count + " catalog row(s), gc'd " + mergeCleaned +267 + " unreferenced merged region(s) and " + splitCleaned +268 + " unreferenced parent region(s)"); +269 } else if (LOG.isTraceEnabled()) { +270 LOG.trace("Scanned " + count + " catalog row(s), gc'd " + mergeCleaned +271 + " unreferenced merged region(s) and " + splitCleaned +272 + " unreferenced parent region(s)"); +273 } +274 return mergeCleaned + splitCleaned; +275 } finally { +276 alreadyRunning.set(false); +277 } +278 } +279 +280 /** +281 * Compare HRegionInfos in a way that has split parents sort BEFORE their +282 * daughters. +283 */ +284 static class SplitParentFirstComparator implements Comparator<HRegionInfo> { +285 Comparator<byte[]> rowEndKeyComparator = new Bytes.RowEndKeyComparator(); +286 @Override +287 public int compare(HRegionInfo left, HRegionInfo right) { +288 // This comparator differs from the one HRegionInfo in that it sorts +289 // parent before daughters. +290 if (left == null) return -1; +291 if (right == null) return 1; +292 // Same table name. +293 int result = left.getTable().compareTo(right.getTable()); +294 if (result != 0) return result; +295 // Compare start keys. +296 result = Bytes.compareTo(left.getStartKey(), right.getStartKey()); +297 if (result != 0) return result; +298 // Compare end keys, but flip the operands so parent comes first +299 result = rowEndKeyComparator.compare(right.getEndKey(), left.getEndKey()); +300 +301 return result; +302 } +303 } +304 +305 /** +306 * If daughters no longer hold reference to the parents, delete the parent. +307 * @param parent HRegionInfo of split offlined parent +308 * @param rowContent Content of <code>parent</code> row in +309 * <code>metaRegionName</code> +310 * @return True if we removed <code>parent</code> from meta table and from +311 * the filesystem. +312 * @throws IOException +313 */ +314 boolean cleanParent(final HRegionInfo parent, Result rowContent) +315 throws IOException { +316 boolean result = false; +317 // Check whether it is a merged region and not clean reference +318 // No necessary to check MERGEB_QUALIFIER because these two qualifiers will +319 // be inserted/deleted together +320 if (rowContent.getValue(HConstants.CATALOG_FAMILY, +321 HConstants.MERGEA_QUALIFIER) != null) { +322 // wait cleaning merge region first +323 return result; +324 } +325 // Run checks on each daughter split. +326 PairOfSameType<HRegionInfo> daughters = MetaTableAccessor.getDaughterRegions(rowContent); +327 Pair<Boolean, Boolean> a = checkDaughterInFs(parent, daughters.getFirst()); +328 Pair<Boolean, Boolean> b = checkDaughterInFs(parent, daughters.getSecond()); +329 if (hasNoReferences(a) && hasNoReferences(b)) { +330 LOG.debug("Deleting region " + parent.getRegionNameAsString() + +331 " because daughter splits no longer hold references"); +332 FileSystem fs = this.services.getMasterFileSystem().getFileSystem(); +333 if (LOG.isTraceEnabled()) LOG.trace("Archiving parent region: " + parent); +334 HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, parent); +335 MetaTableAccessor.deleteRegion(this.connection, parent); +336 result = true; +337 } +338 return result; +339 } +340 +341 /** +342 * @param p A pair where the first boolean says whether or not the daughter +343 * region directory exists in the filesystem and then the second boolean says +344 * whether the daughter has references to the parent. +345 * @return True the passed <code>p</code> signifies no references. +346 */ +347 private boolean hasNoReferences(final Pair<Boolean, Boolean> p) { +348 return !p.getFirst() || !p.getSecond(); +349 } +350 +351 /** +352 * Checks if a daughter region -- either splitA or splitB -- still holds +353 * references to parent. +354 * @param parent Parent region +355 * @param daughter Daughter region +356 * @return A pair where the first boolean says whether or not the daughter +357 * region directory exists in the filesystem and then the second boolean says +358 * whether the daughter has references to the parent. +359 * @throws IOException +360 */ +361 Pair<Boolean, Boolean> checkDaughterInFs(final HRegionInfo parent, final HRegionInfo daughter) +362 throws IOException { +363 if (daughter == null) { +364 return new Pair<Boolean, Boolean>(Boolean.FALSE, Boolean.FALSE); +365 } +366 +367 FileSystem fs = this.services.getMasterFileSystem().getFileSystem(); +368 Path rootdir = this.services.getMasterFileSystem().getRootDir(); +369 Path tabledir = FSUtils.getTableDir(rootdir, daughter.getTable()); +370 +371 Path daughterRegionDir = new Path(tabledir, daughter.getEncodedName()); +372 +373 HRegionFileSystem regionFs = null; +374 +375 try { +376 if (!FSUtils.isExists(fs, daughterRegionDir)) { +377 return new Pair<Boolean, Boolean>(Boolean.FALSE, Boolean.FALSE); +378 } +379 } catch (IOException ioe) { +380 LOG.warn("Error trying to determine if daughter region exists, " + +381 "assuming exists and has references", ioe); +382 return new Pair<Boolean, Boolean>(Boolean.TRUE, Boolean.TRUE); +383 } +384 +385 try { +386 regionFs = HRegionFileSystem.openRegionFromFileSystem( +387 this.services.getConfiguration(), fs, tabledir, daughter, true); +388 } catch (IOException e) { +389 LOG.warn("Error trying to determine referenced files from : " + daughter.getEncodedName() +390 + ", to: " + parent.getEncodedName() + " assuming has references", e); +391 return new Pair<Boolean, Boolean>(Boolean.TRUE, Boolean.TRUE); +392 } +393 +394 boolean references = false; +395 HTableDescriptor parentDescriptor = getTableDescriptor(parent.getTable()); +396 for (HColumnDescriptor family: parentDescriptor.getFamilies()) { +397 if ((references = regionFs.hasReferences(family.getNameAsString()))) { +398 break; +399 } +400 } +401 return new Pair<Boolean, Boolean>(Boolean.TRUE, Boolean.valueOf(references)); +402 } +403 +404 private HTableDescriptor getTableDescriptor(final TableName tableName) +405 throws FileNotFoundException, IOException { +406 return this.services.getTableDescriptors().get(tableName); +407 } +408 +409 /** +410 * Checks if the specified region has merge qualifiers, if so, try to clean +411 * them +412 * @param region +413 * @return true if the specified region doesn't have merge qualifier now +414 * @throws IOException +415 */ +416 public boolean cleanMergeQualifier(final HRegionInfo region) +417 throws IOException { +418 // Get merge regions if it is a merged region and already has merge +419 // qualifier +420 Pair<HRegionInfo, HRegionInfo> mergeRegions = MetaTableAccessor +421 .getRegionsFromMergeQualifier(this.services.getConnection(), +422 region.getRegionName()); +423 if (mergeRegions == null +424 || (mergeRegions.getFirst() == null && mergeRegions.getSecond() == null)) { +425 // It doesn't have merge qualifier, no need to clean +426 return true; +427 } +428 // It shouldn't happen, we must insert/delete these two qualifiers together +429 if (mergeRegions.getFirst() == null || mergeRegions.getSecond() == null) { +430 LOG.error("Merged region " + region.getRegionNameAsString() +431 + " has only one merge qualifier in META."); +432 return false; +433 } +434 return cleanMergeRegion(region, mergeRegions.getFirst(), +435 mergeRegions.getSecond()); +436 } +437 }