Return-Path: X-Original-To: apmail-hbase-commits-archive@www.apache.org Delivered-To: apmail-hbase-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id B32B510842 for ; Fri, 30 Aug 2013 17:31:13 +0000 (UTC) Received: (qmail 51227 invoked by uid 500); 30 Aug 2013 17:31:13 -0000 Delivered-To: apmail-hbase-commits-archive@hbase.apache.org Received: (qmail 51158 invoked by uid 500); 30 Aug 2013 17:31:13 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 51151 invoked by uid 99); 30 Aug 2013 17:31:13 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 30 Aug 2013 17:31:13 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 30 Aug 2013 17:31:11 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 4FB872388860; Fri, 30 Aug 2013 17:30:51 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1519030 - in /hbase/trunk/hbase-server/src: main/java/org/apache/hadoop/hbase/master/ main/java/org/apache/hadoop/hbase/master/balancer/ main/java/org/apache/hadoop/hbase/util/ test/java/org/apache/hadoop/hbase/master/ Date: Fri, 30 Aug 2013 17:30:51 -0000 To: commits@hbase.apache.org From: ddas@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20130830173051.4FB872388860@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: ddas Date: Fri Aug 30 17:30:50 2013 New Revision: 1519030 URL: http://svn.apache.org/r1519030 Log: HBASE-9389. Fixes the verification part of favored node assignments Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentVerificationReport.java hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionPlacementMaintainer.java hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/FavoredNodesPlan.java hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSRegionScanner.java hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentVerificationReport.java URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentVerificationReport.java?rev=1519030&r1=1519029&r2=1519030&view=diff ============================================================================== --- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentVerificationReport.java (original) +++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentVerificationReport.java Fri Aug 30 17:30:50 2013 @@ -582,6 +582,48 @@ public class AssignmentVerificationRepor System.out.println("=============================="); } + /** + * Return the unassigned regions + * @return unassigned regions + */ + List getUnassignedRegions() { + return unAssignedRegionsList; + } + + /** + * Return the regions without favored nodes + * @return regions without favored nodes + */ + List getRegionsWithoutValidFavoredNodes() { + return regionsWithoutValidFavoredNodes; + } + + /** + * Return the regions not assigned to its favored nodes + * @return regions not assigned to its favored nodes + */ + List getNonFavoredAssignedRegions() { + return nonFavoredAssignedRegionList; + } + + /** + * Return the number of regions assigned to their favored nodes + * @return number of regions assigned to their favored nodes + */ + int getTotalFavoredAssignments() { + return totalFavoredAssignments; + } + + /** + * Return the number of regions based on the position (primary/secondary/ + * tertiary) assigned to their favored nodes + * @param position + * @return the number of regions + */ + int getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position position) { + return favoredNodes[position.ordinal()]; + } + private void printHServerAddressSet(Set serverSet) { if (serverSet == null) { return ; Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionPlacementMaintainer.java URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionPlacementMaintainer.java?rev=1519030&r1=1519029&r2=1519030&view=diff ============================================================================== --- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionPlacementMaintainer.java (original) +++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionPlacementMaintainer.java Fri Aug 30 17:30:50 2013 @@ -148,10 +148,13 @@ public class RegionPlacementMaintainer { } /** - * Verify the region placement is consistent with the assignment plan; + * Verify the region placement is consistent with the assignment plan + * @param isDetailMode + * @return reports * @throws IOException */ - public void verifyRegionPlacement(boolean isDetailMode) throws IOException { + public List verifyRegionPlacement(boolean isDetailMode) + throws IOException { System.out.println("Start to verify the region assignment and " + "generate the verification report"); // Get the region assignment snapshot @@ -165,6 +168,7 @@ public class RegionPlacementMaintainer { if (this.enforceLocality == true) { regionLocalityMap = FSUtils.getRegionDegreeLocalityMappingFromFS(conf); } + List reports = new ArrayList(); // Iterate all the tables to fill up the verification report for (TableName table : tables) { if (!this.targetTableSet.isEmpty() && @@ -174,7 +178,9 @@ public class RegionPlacementMaintainer { AssignmentVerificationReport report = new AssignmentVerificationReport(); report.fillUp(table, snapshot, regionLocalityMap); report.print(isDetailMode); + reports.add(report); } + return reports; } /** Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/FavoredNodesPlan.java URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/FavoredNodesPlan.java?rev=1519030&r1=1519029&r2=1519030&view=diff ============================================================================== --- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/FavoredNodesPlan.java (original) +++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/FavoredNodesPlan.java Fri Aug 30 17:30:50 2013 @@ -87,7 +87,7 @@ public class FavoredNodesPlan { return null; } for (Position p : Position.values()) { - if (favoredNodes.get(p.ordinal()).equals(server)) { + if (ServerName.isSameHostnameAndPort(favoredNodes.get(p.ordinal()),server)) { return p; } } Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSRegionScanner.java URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSRegionScanner.java?rev=1519030&r1=1519029&r2=1519030&view=diff ============================================================================== --- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSRegionScanner.java (original) +++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSRegionScanner.java Fri Aug 30 17:30:50 2013 @@ -30,6 +30,7 @@ import org.apache.hadoop.fs.BlockLocatio import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HConstants; /** * Thread that walks over the filesystem, and computes the mappings @@ -89,7 +90,8 @@ class FSRegionScanner implements Runnabl // skip because this is not a CF directory continue; } - if (cfStatus.getPath().getName().startsWith(".")) { + if (cfStatus.getPath().getName().startsWith(".") || + HConstants.HBASE_NON_USER_TABLE_DIRS.contains(cfStatus.getPath().getName())) { continue; } FileStatus[] storeFileLists = fs.listStatus(cfStatus.getPath()); Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java?rev=1519030&r1=1519029&r2=1519030&view=diff ============================================================================== --- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java (original) +++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java Fri Aug 30 17:30:50 2013 @@ -1785,9 +1785,9 @@ public abstract class FSUtils { Path queryPath; // The table files are in ${hbase.rootdir}/data///* if (null == desiredTable) { - queryPath = new Path(new Path(rootPath, HConstants.BASE_NAMESPACE_DIR), "/*/*/"); + queryPath = new Path(new Path(rootPath, HConstants.BASE_NAMESPACE_DIR).toString() + "/*/*/*/"); } else { - queryPath = new Path(FSUtils.getTableDir(rootPath, TableName.valueOf(desiredTable)), "/*/"); + queryPath = new Path(FSUtils.getTableDir(rootPath, TableName.valueOf(desiredTable)).toString() + "/*/"); } // reject all paths that are not appropriate @@ -1805,11 +1805,6 @@ public abstract class FSUtils { return false; } - // not part of a table? - if (!parent.getName().equals(TableName.META_TABLE_NAME.getQualifierAsString())) { - return false; - } - String regionName = path.getName(); if (null == regionName) { return false; Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java?rev=1519030&r1=1519029&r2=1519030&view=diff ============================================================================== --- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java (original) +++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java Fri Aug 30 17:30:50 2013 @@ -210,7 +210,8 @@ public class TestRegionPlacement { @Test public void testRegionPlacement() throws Exception { - byte[] table = Bytes.toBytes("testRegionAssignment"); + String tableStr = "testRegionAssignment"; + byte[] table = Bytes.toBytes(tableStr); // Create a table with REGION_NUM regions. createTable(table, REGION_NUM); @@ -249,8 +250,36 @@ public class TestRegionPlacement { rp.updateAssignmentPlan(shuffledPlan); verifyRegionAssignment(shuffledPlan, REGION_NUM, REGION_NUM); + + // also verify that the AssignmentVerificationReport has the correct information + RegionPlacementMaintainer rp = new RegionPlacementMaintainer(TEST_UTIL.getConfiguration()); + // we are interested in only one table (and hence one report) + rp.setTargetTableName(new String[]{tableStr}); + List reports = rp.verifyRegionPlacement(false); + AssignmentVerificationReport report = reports.get(0); + assertTrue(report.getRegionsWithoutValidFavoredNodes().size() == 0); + assertTrue(report.getNonFavoredAssignedRegions().size() == 0); + assertTrue(report.getTotalFavoredAssignments() >= REGION_NUM); + assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) != 0); + assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) == 0); + assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY) == 0); + assertTrue(report.getUnassignedRegions().size() == 0); + // Check when a RS stops, the regions get assigned to their secondary/tertiary killRandomServerAndVerifyAssignment(); + + // also verify that the AssignmentVerificationReport has the correct information + reports = rp.verifyRegionPlacement(false); + report = reports.get(0); + assertTrue(report.getRegionsWithoutValidFavoredNodes().size() == 0); + assertTrue(report.getNonFavoredAssignedRegions().size() == 0); + assertTrue(report.getTotalFavoredAssignments() >= REGION_NUM); + assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) > 0 && + (report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) > 0 + || report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY) > 0)); + assertTrue((report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) + + report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) + + report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY)) == REGION_NUM); RegionPlacementMaintainer.printAssignmentPlan(currentPlan); }