Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id D4FA2200CBE for ; Thu, 22 Jun 2017 17:01:27 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id D3FB4160BF9; Thu, 22 Jun 2017 15:01:27 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 7E10C160BFD for ; Thu, 22 Jun 2017 17:01:25 +0200 (CEST) Received: (qmail 70962 invoked by uid 500); 22 Jun 2017 15:01:23 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 69952 invoked by uid 99); 22 Jun 2017 15:01:23 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 22 Jun 2017 15:01:23 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 8D04FE9658; Thu, 22 Jun 2017 15:01:21 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: git-site-role@apache.org To: commits@hbase.apache.org Date: Thu, 22 Jun 2017 15:01:32 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [12/17] hbase-site git commit: Published site at 82d554e3783372cc6b05489452c815b57c06f6cd. archived-at: Thu, 22 Jun 2017 15:01:28 -0000 http://git-wip-us.apache.org/repos/asf/hbase-site/blob/1efd3d47/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.AssignRegionAction.html ---------------------------------------------------------------------- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.AssignRegionAction.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.AssignRegionAction.html index d0f1508..085701c 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.AssignRegionAction.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.AssignRegionAction.html @@ -922,7 +922,7 @@ 914 private Comparator<Integer> numRegionsComparator = new Comparator<Integer>() { 915 @Override 916 public int compare(Integer integer, Integer integer2) { -917 return Integer.valueOf(getNumRegions(integer)).compareTo(getNumRegions(integer2)); +917 return Integer.compare(getNumRegions(integer), getNumRegions(integer2)); 918 } 919 }; 920 @@ -937,797 +937,789 @@ 929 private Comparator<Integer> localityComparator = new Comparator<Integer>() { 930 @Override 931 public int compare(Integer integer, Integer integer2) { -932 float locality1 = getLocality(integer); -933 float locality2 = getLocality(integer2); -934 if (locality1 < locality2) { -935 return -1; -936 } else if (locality1 > locality2) { -937 return 1; -938 } else { -939 return 0; -940 } -941 } -942 }; -943 -944 int getLowestLocalityRegionServer() { -945 if (regionFinder == null) { -946 return -1; -947 } else { -948 sortServersByLocality(); -949 // We want to find server with non zero regions having lowest locality. -950 int i = 0; -951 int lowestLocalityServerIndex = serverIndicesSortedByLocality[i]; -952 while (localityPerServer[lowestLocalityServerIndex] == 0 -953 && (regionsPerServer[lowestLocalityServerIndex].length == 0)) { -954 i++; -955 lowestLocalityServerIndex = serverIndicesSortedByLocality[i]; -956 } -957 if (LOG.isTraceEnabled()) { -958 LOG.trace("Lowest locality region server with non zero regions is " -959 + servers[lowestLocalityServerIndex].getHostname() + " with locality " -960 + localityPerServer[lowestLocalityServerIndex]); -961 } -962 return lowestLocalityServerIndex; -963 } -964 } -965 -966 int getLowestLocalityRegionOnServer(int serverIndex) { -967 if (regionFinder != null) { -968 float lowestLocality = 1.0f; -969 int lowestLocalityRegionIndex = -1; -970 if (regionsPerServer[serverIndex].length == 0) { -971 // No regions on that region server -972 return -1; -973 } -974 for (int j = 0; j < regionsPerServer[serverIndex].length; j++) { -975 int regionIndex = regionsPerServer[serverIndex][j]; -976 HDFSBlocksDistribution distribution = regionFinder -977 .getBlockDistribution(regions[regionIndex]); -978 float locality = distribution.getBlockLocalityIndex(servers[serverIndex].getHostname()); -979 // skip empty region -980 if (distribution.getUniqueBlocksTotalWeight() == 0) { -981 continue; -982 } -983 if (locality < lowestLocality) { -984 lowestLocality = locality; -985 lowestLocalityRegionIndex = j; -986 } -987 } -988 if (lowestLocalityRegionIndex == -1) { -989 return -1; -990 } -991 if (LOG.isTraceEnabled()) { -992 LOG.trace("Lowest locality region is " -993 + regions[regionsPerServer[serverIndex][lowestLocalityRegionIndex]] -994 .getRegionNameAsString() + " with locality " + lowestLocality -995 + " and its region server contains " + regionsPerServer[serverIndex].length -996 + " regions"); -997 } -998 return regionsPerServer[serverIndex][lowestLocalityRegionIndex]; -999 } else { -1000 return -1; -1001 } -1002 } -1003 -1004 float getLocalityOfRegion(int region, int server) { -1005 if (regionFinder != null) { -1006 HDFSBlocksDistribution distribution = regionFinder.getBlockDistribution(regions[region]); -1007 return distribution.getBlockLocalityIndex(servers[server].getHostname()); -1008 } else { -1009 return 0f; -1010 } -1011 } -1012 -1013 /** -1014 * Returns a least loaded server which has better locality for this region -1015 * than the current server. -1016 */ -1017 int getLeastLoadedTopServerForRegion(int region, int currentServer) { -1018 if (regionFinder != null) { -1019 List<ServerName> topLocalServers = regionFinder.getTopBlockLocations(regions[region], -1020 servers[currentServer].getHostname()); -1021 int leastLoadedServerIndex = -1; -1022 int load = Integer.MAX_VALUE; -1023 for (ServerName sn : topLocalServers) { -1024 if (!serversToIndex.containsKey(sn.getHostAndPort())) { -1025 continue; -1026 } -1027 int index = serversToIndex.get(sn.getHostAndPort()); -1028 if (regionsPerServer[index] == null) { -1029 continue; -1030 } -1031 int tempLoad = regionsPerServer[index].length; -1032 if (tempLoad <= load) { -1033 leastLoadedServerIndex = index; -1034 load = tempLoad; -1035 } -1036 } -1037 if (leastLoadedServerIndex != -1) { -1038 if (LOG.isTraceEnabled()) { -1039 LOG.trace("Pick the least loaded server " + -1040 servers[leastLoadedServerIndex].getHostname() + -1041 " with better locality for region " + regions[region].getShortNameToLog()); -1042 } -1043 } -1044 return leastLoadedServerIndex; -1045 } else { -1046 return -1; -1047 } -1048 } -1049 -1050 void calculateRegionServerLocalities() { -1051 if (regionFinder == null) { -1052 LOG.warn("Region location finder found null, skipping locality calculations."); -1053 return; -1054 } -1055 for (int i = 0; i < regionsPerServer.length; i++) { -1056 HDFSBlocksDistribution distribution = new HDFSBlocksDistribution(); -1057 if (regionsPerServer[i].length > 0) { -1058 for (int j = 0; j < regionsPerServer[i].length; j++) { -1059 int regionIndex = regionsPerServer[i][j]; -1060 distribution.add(regionFinder.getBlockDistribution(regions[regionIndex])); -1061 } -1062 } else { -1063 LOG.debug("Server " + servers[i].getHostname() + " had 0 regions."); -1064 } -1065 localityPerServer[i] = distribution.getBlockLocalityIndex(servers[i].getHostname()); -1066 } -1067 } -1068 -1069 @VisibleForTesting -1070 protected void setNumRegions(int numRegions) { -1071 this.numRegions = numRegions; -1072 } -1073 -1074 @VisibleForTesting -1075 protected void setNumMovedRegions(int numMovedRegions) { -1076 this.numMovedRegions = numMovedRegions; -1077 } -1078 -1079 @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="SBSC_USE_STRINGBUFFER_CONCATENATION", -1080 justification="Not important but should be fixed") -1081 @Override -1082 public String toString() { -1083 String desc = "Cluster{" + -1084 "servers=["; -1085 for(ServerName sn:servers) { -1086 desc += sn.getHostAndPort() + ", "; +932 return Float.compare(getLocality(integer), getLocality(integer2)); +933 } +934 }; +935 +936 int getLowestLocalityRegionServer() { +937 if (regionFinder == null) { +938 return -1; +939 } else { +940 sortServersByLocality(); +941 // We want to find server with non zero regions having lowest locality. +942 int i = 0; +943 int lowestLocalityServerIndex = serverIndicesSortedByLocality[i]; +944 while (localityPerServer[lowestLocalityServerIndex] == 0 +945 && (regionsPerServer[lowestLocalityServerIndex].length == 0)) { +946 i++; +947 lowestLocalityServerIndex = serverIndicesSortedByLocality[i]; +948 } +949 if (LOG.isTraceEnabled()) { +950 LOG.trace("Lowest locality region server with non zero regions is " +951 + servers[lowestLocalityServerIndex].getHostname() + " with locality " +952 + localityPerServer[lowestLocalityServerIndex]); +953 } +954 return lowestLocalityServerIndex; +955 } +956 } +957 +958 int getLowestLocalityRegionOnServer(int serverIndex) { +959 if (regionFinder != null) { +960 float lowestLocality = 1.0f; +961 int lowestLocalityRegionIndex = -1; +962 if (regionsPerServer[serverIndex].length == 0) { +963 // No regions on that region server +964 return -1; +965 } +966 for (int j = 0; j < regionsPerServer[serverIndex].length; j++) { +967 int regionIndex = regionsPerServer[serverIndex][j]; +968 HDFSBlocksDistribution distribution = regionFinder +969 .getBlockDistribution(regions[regionIndex]); +970 float locality = distribution.getBlockLocalityIndex(servers[serverIndex].getHostname()); +971 // skip empty region +972 if (distribution.getUniqueBlocksTotalWeight() == 0) { +973 continue; +974 } +975 if (locality < lowestLocality) { +976 lowestLocality = locality; +977 lowestLocalityRegionIndex = j; +978 } +979 } +980 if (lowestLocalityRegionIndex == -1) { +981 return -1; +982 } +983 if (LOG.isTraceEnabled()) { +984 LOG.trace("Lowest locality region is " +985 + regions[regionsPerServer[serverIndex][lowestLocalityRegionIndex]] +986 .getRegionNameAsString() + " with locality " + lowestLocality +987 + " and its region server contains " + regionsPerServer[serverIndex].length +988 + " regions"); +989 } +990 return regionsPerServer[serverIndex][lowestLocalityRegionIndex]; +991 } else { +992 return -1; +993 } +994 } +995 +996 float getLocalityOfRegion(int region, int server) { +997 if (regionFinder != null) { +998 HDFSBlocksDistribution distribution = regionFinder.getBlockDistribution(regions[region]); +999 return distribution.getBlockLocalityIndex(servers[server].getHostname()); +1000 } else { +1001 return 0f; +1002 } +1003 } +1004 +1005 /** +1006 * Returns a least loaded server which has better locality for this region +1007 * than the current server. +1008 */ +1009 int getLeastLoadedTopServerForRegion(int region, int currentServer) { +1010 if (regionFinder != null) { +1011 List<ServerName> topLocalServers = regionFinder.getTopBlockLocations(regions[region], +1012 servers[currentServer].getHostname()); +1013 int leastLoadedServerIndex = -1; +1014 int load = Integer.MAX_VALUE; +1015 for (ServerName sn : topLocalServers) { +1016 if (!serversToIndex.containsKey(sn.getHostAndPort())) { +1017 continue; +1018 } +1019 int index = serversToIndex.get(sn.getHostAndPort()); +1020 if (regionsPerServer[index] == null) { +1021 continue; +1022 } +1023 int tempLoad = regionsPerServer[index].length; +1024 if (tempLoad <= load) { +1025 leastLoadedServerIndex = index; +1026 load = tempLoad; +1027 } +1028 } +1029 if (leastLoadedServerIndex != -1) { +1030 if (LOG.isTraceEnabled()) { +1031 LOG.trace("Pick the least loaded server " + +1032 servers[leastLoadedServerIndex].getHostname() + +1033 " with better locality for region " + regions[region].getShortNameToLog()); +1034 } +1035 } +1036 return leastLoadedServerIndex; +1037 } else { +1038 return -1; +1039 } +1040 } +1041 +1042 void calculateRegionServerLocalities() { +1043 if (regionFinder == null) { +1044 LOG.warn("Region location finder found null, skipping locality calculations."); +1045 return; +1046 } +1047 for (int i = 0; i < regionsPerServer.length; i++) { +1048 HDFSBlocksDistribution distribution = new HDFSBlocksDistribution(); +1049 if (regionsPerServer[i].length > 0) { +1050 for (int j = 0; j < regionsPerServer[i].length; j++) { +1051 int regionIndex = regionsPerServer[i][j]; +1052 distribution.add(regionFinder.getBlockDistribution(regions[regionIndex])); +1053 } +1054 } else { +1055 LOG.debug("Server " + servers[i].getHostname() + " had 0 regions."); +1056 } +1057 localityPerServer[i] = distribution.getBlockLocalityIndex(servers[i].getHostname()); +1058 } +1059 } +1060 +1061 @VisibleForTesting +1062 protected void setNumRegions(int numRegions) { +1063 this.numRegions = numRegions; +1064 } +1065 +1066 @VisibleForTesting +1067 protected void setNumMovedRegions(int numMovedRegions) { +1068 this.numMovedRegions = numMovedRegions; +1069 } +1070 +1071 @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="SBSC_USE_STRINGBUFFER_CONCATENATION", +1072 justification="Not important but should be fixed") +1073 @Override +1074 public String toString() { +1075 String desc = "Cluster{" + +1076 "servers=["; +1077 for(ServerName sn:servers) { +1078 desc += sn.getHostAndPort() + ", "; +1079 } +1080 desc += +1081 ", serverIndicesSortedByRegionCount="+ +1082 Arrays.toString(serverIndicesSortedByRegionCount) + +1083 ", regionsPerServer=["; +1084 +1085 for (int[]r:regionsPerServer) { +1086 desc += Arrays.toString(r); 1087 } -1088 desc += -1089 ", serverIndicesSortedByRegionCount="+ -1090 Arrays.toString(serverIndicesSortedByRegionCount) + -1091 ", regionsPerServer=["; -1092 -1093 for (int[]r:regionsPerServer) { -1094 desc += Arrays.toString(r); -1095 } -1096 desc += "]" + -1097 ", numMaxRegionsPerTable=" + -1098 Arrays.toString(numMaxRegionsPerTable) + -1099 ", numRegions=" + -1100 numRegions + -1101 ", numServers=" + -1102 numServers + -1103 ", numTables=" + -1104 numTables + -1105 ", numMovedRegions=" + -1106 numMovedRegions + -1107 '}'; -1108 return desc; -1109 } -1110 } -1111 -1112 // slop for regions -1113 protected float slop; -1114 // overallSlop to controll simpleLoadBalancer's cluster level threshold -1115 protected float overallSlop; -1116 protected Configuration config; -1117 protected RackManager rackManager; -1118 private static final Random RANDOM = new Random(System.currentTimeMillis()); -1119 private static final Log LOG = LogFactory.getLog(BaseLoadBalancer.class); -1120 -1121 // Regions of these tables are put on the master by default. -1122 private static final String[] DEFAULT_TABLES_ON_MASTER = -1123 new String[] {AccessControlLists.ACL_TABLE_NAME.getNameAsString(), -1124 TableName.NAMESPACE_TABLE_NAME.getNameAsString(), -1125 TableName.META_TABLE_NAME.getNameAsString()}; -1126 -1127 public static final String TABLES_ON_MASTER = -1128 "hbase.balancer.tablesOnMaster"; -1129 -1130 protected final Set<String> tablesOnMaster = new HashSet<>(); -1131 protected MetricsBalancer metricsBalancer = null; -1132 protected ClusterStatus clusterStatus = null; -1133 protected ServerName masterServerName; -1134 protected MasterServices services; -1135 -1136 /** -1137 * By default, regions of some small system tables such as meta, -1138 * namespace, and acl are assigned to the active master. If you don't -1139 * want to assign any region to the active master, you need to -1140 * configure "hbase.balancer.tablesOnMaster" to "none". -1141 */ -1142 protected static String[] getTablesOnMaster(Configuration conf) { -1143 String valueString = conf.get(TABLES_ON_MASTER); -1144 if (valueString == null) { -1145 return DEFAULT_TABLES_ON_MASTER; -1146 } -1147 valueString = valueString.trim(); -1148 if (valueString.equalsIgnoreCase("none")) { -1149 return null; -1150 } -1151 return StringUtils.getStrings(valueString); +1088 desc += "]" + +1089 ", numMaxRegionsPerTable=" + +1090 Arrays.toString(numMaxRegionsPerTable) + +1091 ", numRegions=" + +1092 numRegions + +1093 ", numServers=" + +1094 numServers + +1095 ", numTables=" + +1096 numTables + +1097 ", numMovedRegions=" + +1098 numMovedRegions + +1099 '}'; +1100 return desc; +1101 } +1102 } +1103 +1104 // slop for regions +1105 protected float slop; +1106 // overallSlop to controll simpleLoadBalancer's cluster level threshold +1107 protected float overallSlop; +1108 protected Configuration config; +1109 protected RackManager rackManager; +1110 private static final Random RANDOM = new Random(System.currentTimeMillis()); +1111 private static final Log LOG = LogFactory.getLog(BaseLoadBalancer.class); +1112 +1113 // Regions of these tables are put on the master by default. +1114 private static final String[] DEFAULT_TABLES_ON_MASTER = +1115 new String[] {AccessControlLists.ACL_TABLE_NAME.getNameAsString(), +1116 TableName.NAMESPACE_TABLE_NAME.getNameAsString(), +1117 TableName.META_TABLE_NAME.getNameAsString()}; +1118 +1119 public static final String TABLES_ON_MASTER = +1120 "hbase.balancer.tablesOnMaster"; +1121 +1122 protected final Set<String> tablesOnMaster = new HashSet<>(); +1123 protected MetricsBalancer metricsBalancer = null; +1124 protected ClusterStatus clusterStatus = null; +1125 protected ServerName masterServerName; +1126 protected MasterServices services; +1127 +1128 /** +1129 * By default, regions of some small system tables such as meta, +1130 * namespace, and acl are assigned to the active master. If you don't +1131 * want to assign any region to the active master, you need to +1132 * configure "hbase.balancer.tablesOnMaster" to "none". +1133 */ +1134 protected static String[] getTablesOnMaster(Configuration conf) { +1135 String valueString = conf.get(TABLES_ON_MASTER); +1136 if (valueString == null) { +1137 return DEFAULT_TABLES_ON_MASTER; +1138 } +1139 valueString = valueString.trim(); +1140 if (valueString.equalsIgnoreCase("none")) { +1141 return null; +1142 } +1143 return StringUtils.getStrings(valueString); +1144 } +1145 +1146 /** +1147 * Check if configured to put any tables on the active master +1148 */ +1149 public static boolean tablesOnMaster(Configuration conf) { +1150 String[] tables = getTablesOnMaster(conf); +1151 return tables != null && tables.length > 0; 1152 } 1153 -1154 /** -1155 * Check if configured to put any tables on the active master -1156 */ -1157 public static boolean tablesOnMaster(Configuration conf) { -1158 String[] tables = getTablesOnMaster(conf); -1159 return tables != null && tables.length > 0; -1160 } -1161 -1162 public static boolean userTablesOnMaster(Configuration conf) { -1163 String[] tables = getTablesOnMaster(conf); -1164 if (tables == null || tables.length == 0) { -1165 return false; -1166 } -1167 for (String tn:tables) { -1168 if (!tn.startsWith("hbase:")) { -1169 return true; -1170 } -1171 } -1172 return false; -1173 } -1174 -1175 @Override -1176 public void setConf(Configuration conf) { -1177 setSlop(conf); -1178 if (slop < 0) slop = 0; -1179 else if (slop > 1) slop = 1; -1180 -1181 if (overallSlop < 0) overallSlop = 0; -1182 else if (overallSlop > 1) overallSlop = 1; -1183 -1184 this.config = conf; -1185 String[] tables = getTablesOnMaster(conf); -1186 if (tables != null && tables.length > 0) { -1187 Collections.addAll(tablesOnMaster, tables); -1188 } -1189 this.rackManager = new RackManager(getConf()); -1190 regionFinder.setConf(conf); -1191 } -1192 -1193 protected void setSlop(Configuration conf) { -1194 this.slop = conf.getFloat("hbase.regions.slop", (float) 0.2); -1195 this.overallSlop = conf.getFloat("hbase.regions.overallSlop", slop); -1196 } -1197 -1198 /** -1199 * Check if a region belongs to some small system table. -1200 * If so, the primary replica may be expected to be put on the master regionserver. +1154 public static boolean userTablesOnMaster(Configuration conf) { +1155 String[] tables = getTablesOnMaster(conf); +1156 if (tables == null || tables.length == 0) { +1157 return false; +1158 } +1159 for (String tn:tables) { +1160 if (!tn.startsWith("hbase:")) { +1161 return true; +1162 } +1163 } +1164 return false; +1165 } +1166 +1167 @Override +1168 public void setConf(Configuration conf) { +1169 setSlop(conf); +1170 if (slop < 0) slop = 0; +1171 else if (slop > 1) slop = 1; +1172 +1173 if (overallSlop < 0) overallSlop = 0; +1174 else if (overallSlop > 1) overallSlop = 1; +1175 +1176 this.config = conf; +1177 String[] tables = getTablesOnMaster(conf); +1178 if (tables != null && tables.length > 0) { +1179 Collections.addAll(tablesOnMaster, tables); +1180 } +1181 this.rackManager = new RackManager(getConf()); +1182 regionFinder.setConf(conf); +1183 } +1184 +1185 protected void setSlop(Configuration conf) { +1186 this.slop = conf.getFloat("hbase.regions.slop", (float) 0.2); +1187 this.overallSlop = conf.getFloat("hbase.regions.overallSlop", slop); +1188 } +1189 +1190 /** +1191 * Check if a region belongs to some small system table. +1192 * If so, the primary replica may be expected to be put on the master regionserver. +1193 */ +1194 public boolean shouldBeOnMaster(HRegionInfo region) { +1195 return tablesOnMaster.contains(region.getTable().getNameAsString()) +1196 && region.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID; +1197 } +1198 +1199 /** +1200 * Balance the regions that should be on master regionserver. 1201 */ -1202 public boolean shouldBeOnMaster(HRegionInfo region) { -1203 return tablesOnMaster.contains(region.getTable().getNameAsString()) -1204 && region.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID; -1205 } -1206 -1207 /** -1208 * Balance the regions that should be on master regionserver. -1209 */ -1210 protected List<RegionPlan> balanceMasterRegions( -1211 Map<ServerName, List<HRegionInfo>> clusterMap) { -1212 if (masterServerName == null -1213 || clusterMap == null || clusterMap.size() <= 1) return null; -1214 List<RegionPlan> plans = null; -1215 List<HRegionInfo> regions = clusterMap.get(masterServerName); -1216 if (regions != null) { -1217 Iterator<ServerName> keyIt = null; -1218 for (HRegionInfo region: regions) { -1219 if (shouldBeOnMaster(region)) continue; -1220 -1221 // Find a non-master regionserver to host the region -1222 if (keyIt == null || !keyIt.hasNext()) { -1223 keyIt = clusterMap.keySet().iterator(); -1224 } -1225 ServerName dest = keyIt.next(); -1226 if (masterServerName.equals(dest)) { -1227 if (!keyIt.hasNext()) { -1228 keyIt = clusterMap.keySet().iterator(); -1229 } -1230 dest = keyIt.next(); -1231 } -1232 -1233 // Move this region away from the master regionserver -1234 RegionPlan plan = new RegionPlan(region, masterServerName, dest); -1235 if (plans == null) { -1236 plans = new ArrayList<>(); -1237 } -1238 plans.add(plan); -1239 } -1240 } -1241 for (Map.Entry<ServerName, List<HRegionInfo>> server: clusterMap.entrySet()) { -1242 if (masterServerName.equals(server.getKey())) continue; -1243 for (HRegionInfo region: server.getValue()) { -1244 if (!shouldBeOnMaster(region)) continue; -1245 -1246 // Move this region to the master regionserver -1247 RegionPlan plan = new RegionPlan(region, server.getKey(), masterServerName); -1248 if (plans == null) { -1249 plans = new ArrayList<>(); -1250 } -1251 plans.add(plan); -1252 } -1253 } -1254 return plans; -1255 } -1256 -1257 /** -1258 * Assign the regions that should be on master regionserver. -1259 */ -1260 protected Map<ServerName, List<HRegionInfo>> assignMasterRegions( -1261 Collection<HRegionInfo> regions, List<ServerName> servers) { -1262 if (servers == null || regions == null || regions.isEmpty()) { -1263 return null; -1264 } -1265 Map<ServerName, List<HRegionInfo>> assignments = new TreeMap<>(); -1266 if (masterServerName != null && servers.contains(masterServerName)) { -1267 assignments.put(masterServerName, new ArrayList<>()); -1268 for (HRegionInfo region: regions) { -1269 if (shouldBeOnMaster(region)) { -1270 assignments.get(masterServerName).add(region); -1271 } -1272 } -1273 } -1274 return assignments; -1275 } -1276 -1277 @Override -1278 public Configuration getConf() { -1279 return this.config; -1280 } -1281 -1282 @Override -1283 public synchronized void setClusterStatus(ClusterStatus st) { -1284 this.clusterStatus = st; -1285 regionFinder.setClusterStatus(st); -1286 } -1287 -1288 @Override -1289 public void setClusterLoad(Map<TableName, Map<ServerName, List<HRegionInfo>>> clusterLoad){ -1290 -1291 } -1292 -1293 @Override -1294 public void setMasterServices(MasterServices masterServices) { -1295 masterServerName = masterServices.getServerName(); -1296 this.services = masterServices; -1297 this.regionFinder.setServices(masterServices); -1298 } -1299 -1300 public void setRackManager(RackManager rackManager) { -1301 this.rackManager = rackManager; -1302 } -1303 -1304 protected boolean needsBalance(Cluster c) { -1305 ClusterLoadState cs = new ClusterLoadState(c.clusterState); -1306 if (cs.getNumServers() < MIN_SERVER_BALANCE) { -1307 if (LOG.isDebugEnabled()) { -1308 LOG.debug("Not running balancer because only " + cs.getNumServers() -1309 + " active regionserver(s)"); -1310 } -1311 return false; -1312 } -1313 if(areSomeRegionReplicasColocated(c)) return true; -1314 // Check if we even need to do any load balancing -1315 // HBASE-3681 check sloppiness first -1316 float average = cs.getLoadAverage(); // for logging -1317 int floor = (int) Math.floor(average * (1 - slop)); -1318 int ceiling = (int) Math.ceil(average * (1 + slop)); -1319 if (!(cs.getMaxLoad() > ceiling || cs.getMinLoad() < floor)) { -1320 NavigableMap<ServerAndLoad, List<HRegionInfo>> serversByLoad = cs.getServersByLoad(); -1321 if (LOG.isTraceEnabled()) { -1322 // If nothing to balance, then don't say anything unless trace-level logging. -1323 LOG.trace("Skipping load balancing because balanced cluster; " + -1324 "servers=" + cs.getNumServers() + -1325 " regions=" + cs.getNumRegions() + " average=" + average + -1326 " mostloaded=" + serversByLoad.lastKey().getLoad() + -1327 " leastloaded=" + serversByLoad.firstKey().getLoad()); -1328 } -1329 return false; -1330 } -1331 return true; -1332 } -1333 -1334 /** -1335 * Subclasses should implement this to return true if the cluster has nodes that hosts -1336 * multiple replicas for the same region, or, if there are multiple racks and the same -1337 * rack hosts replicas of the same region -1338 * @param c Cluster information -1339 * @return whether region replicas are currently co-located -1340 */ -1341 protected boolean areSomeRegionReplicasColocated(Cluster c) { -1342 return false; -1343 } -1344 -1345 /** -1346 * Generates a bulk assignment plan to be used on cluster startup using a -1347 * simple round-robin assignment. -1348 * <p> -1349 * Takes a list of all the regions and all the servers in the cluster and -1350 * returns a map of each server to the regions that it should be assigned. -1351 * <p> -1352 * Currently implemented as a round-robin assignment. Same invariant as load -1353 * balancing, all servers holding floor(avg) or ceiling(avg). -1354 * -1355 * TODO: Use block locations from HDFS to place regions with their blocks -1356 * -1357 * @param regions all regions -1358 * @param servers all servers -1359 * @return map of server to the regions it should take, or null if no -1360 * assignment is possible (ie. no regions or no servers) -1361 */ -1362 @Override -1363 public Map<ServerName, List<HRegionInfo>> roundRobinAssignment(List<HRegionInfo> regions, -1364 List<ServerName> servers) throws HBaseIOException { -1365 metricsBalancer.incrMiscInvocations(); -1366 Map<ServerName, List<HRegionInfo>> assignments = assignMasterRegions(regions, servers); -1367 if (assignments != null && !assignments.isEmpty()) { -1368 servers = new ArrayList<>(servers); -1369 // Guarantee not to put other regions on master -1370 servers.remove(masterServerName); -1371 List<HRegionInfo> masterRegions = assignments.get(masterServerName); -1372 if (!masterRegions.isEmpty()) { -1373 regions = new ArrayList<>(regions); -1374 for (HRegionInfo region: masterRegions) { -1375 regions.remove(region); -1376 } -1377 } -1378 } -1379 if (regions == null || regions.isEmpty()) { -1380 return assignments; -1381 } -1382 -1383 int numServers = servers == null ? 0 : servers.size(); -1384 if (numServers == 0) { -1385 LOG.warn("Wanted to do round robin assignment but no servers to assign to"); -1386 return null; -1387 } -1388 -1389 // TODO: instead of retainAssignment() and roundRobinAssignment(), we should just run the -1390 // normal LB.balancerCluster() with unassignedRegions. We only need to have a candidate -1391 // generator for AssignRegionAction. The LB will ensure the regions are mostly local -1392 // and balanced. This should also run fast with fewer number of iterations. -1393 -1394 if (numServers == 1) { // Only one server, nothing fancy we can do here -1395 ServerName server = servers.get(0); -1396 assignments.put(server, new ArrayList<>(regions)); -1397 return assignments; -1398 } -1399 -1400 Cluster cluster = createCluster(servers, regions, false); -1401 List<HRegionInfo> unassignedRegions = new ArrayList<>(); -1402 -1403 roundRobinAssignment(cluster, regions, unassignedRegions, -1404 servers, assignments); -1405 -1406 List<HRegionInfo> lastFewRegions = new ArrayList<>(); -1407 // assign the remaining by going through the list and try to assign to servers one-by-one -1408 int serverIdx = RANDOM.nextInt(numServers); -1409 for (HRegionInfo region : unassignedRegions) { -1410 boolean assigned = false; -1411 for (int j = 0; j < numServers; j++) { // try all servers one by one -1412 ServerName serverName = servers.get((j + serverIdx) % numServers); -1413 if (!cluster.wouldLowerAvailability(region, serverName)) { -1414 List<HRegionInfo> serverRegions = assignments.get(serverName); -1415 if (serverRegions == null) { -1416 serverRegions = new ArrayList<>(); -1417 assignments.put(serverName, serverRegions); -1418 } -1419 serverRegions.add(region); -1420 cluster.doAssignRegion(region, serverName); -1421 serverIdx = (j + serverIdx + 1) % numServers; //remain from next server -1422 assigned = true; -1423 break; -1424 } -1425 } -1426 if (!assigned) { -1427 lastFewRegions.add(region); -1428 } -1429 } -1430 // just sprinkle the rest of the regions on random regionservers. The balanceCluster will -1431 // make it optimal later. we can end up with this if numReplicas > numServers. -1432 for (HRegionInfo region : lastFewRegions) { -1433 int i = RANDOM.nextInt(numServers); -1434 ServerName server = servers.get(i); -1435 List<HRegionInfo> serverRegions = assignments.get(server); -1436 if (serverRegions == null) { -1437 serverRegions = new ArrayList<>(); -1438 assignments.put(server, serverRegions); -1439 } -1440 serverRegions.add(region); -1441 cluster.doAssignRegion(region, server); +1202 protected List<RegionPlan> balanceMasterRegions( +1203 Map<ServerName, List<HRegionInfo>> clusterMap) { +1204 if (masterServerName == null +1205 || clusterMap == null || clusterMap.size() <= 1) return null; +1206 List<RegionPlan> plans = null; +1207 List<HRegionInfo> regions = clusterMap.get(masterServerName); +1208 if (regions != null) { +1209 Iterator<ServerName> keyIt = null; +1210 for (HRegionInfo region: regions) { +1211 if (shouldBeOnMaster(region)) continue; +1212 +1213 // Find a non-master regionserver to host the region +1214 if (keyIt == null || !keyIt.hasNext()) { +1215 keyIt = clusterMap.keySet().iterator(); +1216 } +1217 ServerName dest = keyIt.next(); +1218 if (masterServerName.equals(dest)) { +1219 if (!keyIt.hasNext()) { +1220 keyIt = clusterMap.keySet().iterator(); +1221 } +1222 dest = keyIt.next(); +1223 } +1224 +1225 // Move this region away from the master regionserver +1226 RegionPlan plan = new RegionPlan(region, masterServerName, dest); +1227 if (plans == null) { +1228 plans = new ArrayList<>(); +1229 } +1230 plans.add(plan); +1231 } +1232 } +1233 for (Map.Entry<ServerName, List<HRegionInfo>> server: clusterMap.entrySet()) { +1234 if (masterServerName.equals(server.getKey())) continue; +1235 for (HRegionInfo region: server.getValue()) { +1236 if (!shouldBeOnMaster(region)) continue; +1237 +1238 // Move this region to the master regionserver +1239 RegionPlan plan = new RegionPlan(region, server.getKey(), masterServerName); +1240 if (plans == null) { +1241 plans = new ArrayList<>(); +1242 } +1243 plans.add(plan); +1244 } +1245 } +1246 return plans; +1247 } +1248 +1249 /** +1250 * Assign the regions that should be on master regionserver. +1251 */ +1252 protected Map<ServerName, List<HRegionInfo>> assignMasterRegions( +1253 Collection<HRegionInfo> regions, List<ServerName> servers) { +1254 if (servers == null || regions == null || regions.isEmpty()) { +1255 return null; +1256 } +1257 Map<ServerName, List<HRegionInfo>> assignments = new TreeMap<>(); +1258 if (masterServerName != null && servers.contains(masterServerName)) { +1259 assignments.put(masterServerName, new ArrayList<>()); +1260 for (HRegionInfo region: regions) { +1261 if (shouldBeOnMaster(region)) { +1262 assignments.get(masterServerName).add(region); +1263 } +1264 } +1265 } +1266 return assignments; +1267 } +1268 +1269 @Override +1270 public Configuration getConf() { +1271 return this.config; +1272 } +1273 +1274 @Override +1275 public synchronized void setClusterStatus(ClusterStatus st) { +1276 this.clusterStatus = st; +1277 regionFinder.setClusterStatus(st); +1278 } +1279 +1280 @Override +1281 public void setClusterLoad(Map<TableName, Map<ServerName, List<HRegionInfo>>> clusterLoad){ +1282 +1283 } +1284 +1285 @Override +1286 public void setMasterServices(MasterServices masterServices) { +1287 masterServerName = masterServices.getServerName(); +1288 this.services = masterServices; +1289 this.regionFinder.setServices(masterServices); +1290 } +1291 +1292 public void setRackManager(RackManager rackManager) { +1293 this.rackManager = rackManager; +1294 } +1295 +1296 protected boolean needsBalance(Cluster c) { +1297 ClusterLoadState cs = new ClusterLoadState(c.clusterState); +1298 if (cs.getNumServers() < MIN_SERVER_BALANCE) { +1299 if (LOG.isDebugEnabled()) { +1300 LOG.debug("Not running balancer because only " + cs.getNumServers() +1301 + " active regionserver(s)"); +1302 } +1303 return false; +1304 } +1305 if(areSomeRegionReplicasColocated(c)) return true; +1306 // Check if we even need to do any load balancing +1307 // HBASE-3681 check sloppiness first +1308 float average = cs