Return-Path: X-Original-To: apmail-hbase-commits-archive@www.apache.org Delivered-To: apmail-hbase-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 71DEE18D53 for ; Thu, 11 Feb 2016 17:34:16 +0000 (UTC) Received: (qmail 26633 invoked by uid 500); 11 Feb 2016 17:34:11 -0000 Delivered-To: apmail-hbase-commits-archive@hbase.apache.org Received: (qmail 26552 invoked by uid 500); 11 Feb 2016 17:34:11 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 23231 invoked by uid 99); 11 Feb 2016 17:34:08 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 11 Feb 2016 17:34:08 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 35A1FE0E4C; Thu, 11 Feb 2016 17:34:08 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: misty@apache.org To: commits@hbase.apache.org Date: Thu, 11 Feb 2016 17:34:34 -0000 Message-Id: <319f6b8a146b4210a56a03033894855c@git.apache.org> In-Reply-To: <33ea69418bfa4b81897bd860a19e4c52@git.apache.org> References: <33ea69418bfa4b81897bd860a19e4c52@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [28/51] [partial] hbase-site git commit: Published site at 29a192ef3cbe3b9cc12a6ee38f39e1199ac9790f. http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8bb348c6/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.html ---------------------------------------------------------------------- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.html index ccccc8d..d2f53d2 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.html @@ -89,16 +89,16 @@ 081 return UNKNOWN_RACK; 082 } 083 } -084 +084 085 /** 086 * The constructor that uses the basic MetricsBalancer 087 */ 088 protected BaseLoadBalancer() { 089 metricsBalancer = new MetricsBalancer(); 090 } -091 +091 092 /** -093 * This Constructor accepts an instance of MetricsBalancer, +093 * This Constructor accepts an instance of MetricsBalancer, 094 * which will be used instead of creating a new one 095 */ 096 protected BaseLoadBalancer(MetricsBalancer metricsBalancer) { @@ -1287,284 +1287,251 @@ 1279 } 1280 1281 /** -1282 * Generates an immediate assignment plan to be used by a new master for -1283 * regions in transition that do not have an already known destination. -1284 * -1285 * Takes a list of regions that need immediate assignment and a list of all -1286 * available servers. Returns a map of regions to the server they should be -1287 * assigned to. -1288 * -1289 * This method will return quickly and does not do any intelligent balancing. -1290 * The goal is to make a fast decision not the best decision possible. -1291 * -1292 * Currently this is random. -1293 * -1294 * @param regions -1295 * @param servers -1296 * @return map of regions to the server it should be assigned to -1297 */ -1298 @Override -1299 public Map<HRegionInfo, ServerName> immediateAssignment(List<HRegionInfo> regions, -1300 List<ServerName> servers) { -1301 metricsBalancer.incrMiscInvocations(); -1302 if (servers == null || servers.isEmpty()) { -1303 LOG.warn("Wanted to do random assignment but no servers to assign to"); -1304 return null; -1305 } -1306 -1307 Map<HRegionInfo, ServerName> assignments = new TreeMap<HRegionInfo, ServerName>(); -1308 for (HRegionInfo region : regions) { -1309 assignments.put(region, randomAssignment(region, servers)); -1310 } -1311 return assignments; -1312 } -1313 -1314 /** -1315 * Used to assign a single region to a random server. -1316 */ -1317 @Override -1318 public ServerName randomAssignment(HRegionInfo regionInfo, List<ServerName> servers) { -1319 metricsBalancer.incrMiscInvocations(); -1320 if (servers != null && servers.contains(masterServerName)) { -1321 if (shouldBeOnMaster(regionInfo)) { -1322 return masterServerName; -1323 } -1324 servers = new ArrayList<ServerName>(servers); -1325 // Guarantee not to put other regions on master -1326 servers.remove(masterServerName); -1327 } -1328 -1329 int numServers = servers == null ? 0 : servers.size(); -1330 if (numServers == 0) { -1331 LOG.warn("Wanted to do retain assignment but no servers to assign to"); -1332 return null; -1333 } -1334 if (numServers == 1) { // Only one server, nothing fancy we can do here -1335 return servers.get(0); -1336 } -1337 -1338 List<HRegionInfo> regions = Lists.newArrayList(regionInfo); -1339 Cluster cluster = createCluster(servers, regions); -1340 return randomAssignment(cluster, regionInfo, servers); -1341 } -1342 -1343 /** -1344 * Generates a bulk assignment startup plan, attempting to reuse the existing -1345 * assignment information from META, but adjusting for the specified list of -1346 * available/online servers available for assignment. -1347 * <p> -1348 * Takes a map of all regions to their existing assignment from META. Also -1349 * takes a list of online servers for regions to be assigned to. Attempts to -1350 * retain all assignment, so in some instances initial assignment will not be -1351 * completely balanced. -1352 * <p> -1353 * Any leftover regions without an existing server to be assigned to will be -1354 * assigned randomly to available servers. -1355 * -1356 * @param regions regions and existing assignment from meta -1357 * @param servers available servers -1358 * @return map of servers and regions to be assigned to them -1359 */ -1360 @Override -1361 public Map<ServerName, List<HRegionInfo>> retainAssignment(Map<HRegionInfo, ServerName> regions, -1362 List<ServerName> servers) { -1363 // Update metrics -1364 metricsBalancer.incrMiscInvocations(); -1365 Map<ServerName, List<HRegionInfo>> assignments -1366 = assignMasterRegions(regions.keySet(), servers); -1367 if (assignments != null && !assignments.isEmpty()) { -1368 servers = new ArrayList<ServerName>(servers); -1369 // Guarantee not to put other regions on master -1370 servers.remove(masterServerName); -1371 List<HRegionInfo> masterRegions = assignments.get(masterServerName); -1372 if (!masterRegions.isEmpty()) { -1373 regions = new HashMap<HRegionInfo, ServerName>(regions); -1374 for (HRegionInfo region: masterRegions) { -1375 regions.remove(region); -1376 } -1377 } -1378 } -1379 if (regions == null || regions.isEmpty()) { -1380 return assignments; -1381 } +1282 * Used to assign a single region to a random server. +1283 */ +1284 @Override +1285 public ServerName randomAssignment(HRegionInfo regionInfo, List<ServerName> servers) { +1286 metricsBalancer.incrMiscInvocations(); +1287 if (servers != null && servers.contains(masterServerName)) { +1288 if (shouldBeOnMaster(regionInfo)) { +1289 return masterServerName; +1290 } +1291 servers = new ArrayList<ServerName>(servers); +1292 // Guarantee not to put other regions on master +1293 servers.remove(masterServerName); +1294 } +1295 +1296 int numServers = servers == null ? 0 : servers.size(); +1297 if (numServers == 0) { +1298 LOG.warn("Wanted to do retain assignment but no servers to assign to"); +1299 return null; +1300 } +1301 if (numServers == 1) { // Only one server, nothing fancy we can do here +1302 return servers.get(0); +1303 } +1304 +1305 List<HRegionInfo> regions = Lists.newArrayList(regionInfo); +1306 Cluster cluster = createCluster(servers, regions); +1307 return randomAssignment(cluster, regionInfo, servers); +1308 } +1309 +1310 /** +1311 * Generates a bulk assignment startup plan, attempting to reuse the existing +1312 * assignment information from META, but adjusting for the specified list of +1313 * available/online servers available for assignment. +1314 * <p> +1315 * Takes a map of all regions to their existing assignment from META. Also +1316 * takes a list of online servers for regions to be assigned to. Attempts to +1317 * retain all assignment, so in some instances initial assignment will not be +1318 * completely balanced. +1319 * <p> +1320 * Any leftover regions without an existing server to be assigned to will be +1321 * assigned randomly to available servers. +1322 * +1323 * @param regions regions and existing assignment from meta +1324 * @param servers available servers +1325 * @return map of servers and regions to be assigned to them +1326 */ +1327 @Override +1328 public Map<ServerName, List<HRegionInfo>> retainAssignment(Map<HRegionInfo, ServerName> regions, +1329 List<ServerName> servers) { +1330 // Update metrics +1331 metricsBalancer.incrMiscInvocations(); +1332 Map<ServerName, List<HRegionInfo>> assignments +1333 = assignMasterRegions(regions.keySet(), servers); +1334 if (assignments != null && !assignments.isEmpty()) { +1335 servers = new ArrayList<ServerName>(servers); +1336 // Guarantee not to put other regions on master +1337 servers.remove(masterServerName); +1338 List<HRegionInfo> masterRegions = assignments.get(masterServerName); +1339 if (!masterRegions.isEmpty()) { +1340 regions = new HashMap<HRegionInfo, ServerName>(regions); +1341 for (HRegionInfo region: masterRegions) { +1342 regions.remove(region); +1343 } +1344 } +1345 } +1346 if (regions == null || regions.isEmpty()) { +1347 return assignments; +1348 } +1349 +1350 int numServers = servers == null ? 0 : servers.size(); +1351 if (numServers == 0) { +1352 LOG.warn("Wanted to do retain assignment but no servers to assign to"); +1353 return null; +1354 } +1355 if (numServers == 1) { // Only one server, nothing fancy we can do here +1356 ServerName server = servers.get(0); +1357 assignments.put(server, new ArrayList<HRegionInfo>(regions.keySet())); +1358 return assignments; +1359 } +1360 +1361 // Group all of the old assignments by their hostname. +1362 // We can't group directly by ServerName since the servers all have +1363 // new start-codes. +1364 +1365 // Group the servers by their hostname. It's possible we have multiple +1366 // servers on the same host on different ports. +1367 ArrayListMultimap<String, ServerName> serversByHostname = ArrayListMultimap.create(); +1368 for (ServerName server : servers) { +1369 assignments.put(server, new ArrayList<HRegionInfo>()); +1370 serversByHostname.put(server.getHostname(), server); +1371 } +1372 +1373 // Collection of the hostnames that used to have regions +1374 // assigned, but for which we no longer have any RS running +1375 // after the cluster restart. +1376 Set<String> oldHostsNoLongerPresent = Sets.newTreeSet(); +1377 +1378 int numRandomAssignments = 0; +1379 int numRetainedAssigments = 0; +1380 +1381 Cluster cluster = createCluster(servers, regions.keySet()); 1382 -1383 int numServers = servers == null ? 0 : servers.size(); -1384 if (numServers == 0) { -1385 LOG.warn("Wanted to do retain assignment but no servers to assign to"); -1386 return null; -1387 } -1388 if (numServers == 1) { // Only one server, nothing fancy we can do here -1389 ServerName server = servers.get(0); -1390 assignments.put(server, new ArrayList<HRegionInfo>(regions.keySet())); -1391 return assignments; -1392 } -1393 -1394 // Group all of the old assignments by their hostname. -1395 // We can't group directly by ServerName since the servers all have -1396 // new start-codes. -1397 -1398 // Group the servers by their hostname. It's possible we have multiple -1399 // servers on the same host on different ports. -1400 ArrayListMultimap<String, ServerName> serversByHostname = ArrayListMultimap.create(); -1401 for (ServerName server : servers) { -1402 assignments.put(server, new ArrayList<HRegionInfo>()); -1403 serversByHostname.put(server.getHostname(), server); -1404 } -1405 -1406 // Collection of the hostnames that used to have regions -1407 // assigned, but for which we no longer have any RS running -1408 // after the cluster restart. -1409 Set<String> oldHostsNoLongerPresent = Sets.newTreeSet(); -1410 -1411 int numRandomAssignments = 0; -1412 int numRetainedAssigments = 0; -1413 -1414 Cluster cluster = createCluster(servers, regions.keySet()); -1415 -1416 for (Map.Entry<HRegionInfo, ServerName> entry : regions.entrySet()) { -1417 HRegionInfo region = entry.getKey(); -1418 ServerName oldServerName = entry.getValue(); -1419 List<ServerName> localServers = new ArrayList<ServerName>(); -1420 if (oldServerName != null) { -1421 localServers = serversByHostname.get(oldServerName.getHostname()); +1383 for (Map.Entry<HRegionInfo, ServerName> entry : regions.entrySet()) { +1384 HRegionInfo region = entry.getKey(); +1385 ServerName oldServerName = entry.getValue(); +1386 List<ServerName> localServers = new ArrayList<ServerName>(); +1387 if (oldServerName != null) { +1388 localServers = serversByHostname.get(oldServerName.getHostname()); +1389 } +1390 if (localServers.isEmpty()) { +1391 // No servers on the new cluster match up with this hostname, +1392 // assign randomly. +1393 ServerName randomServer = randomAssignment(cluster, region, servers); +1394 assignments.get(randomServer).add(region); +1395 numRandomAssignments++; +1396 if (oldServerName != null) oldHostsNoLongerPresent.add(oldServerName.getHostname()); +1397 } else if (localServers.size() == 1) { +1398 // the usual case - one new server on same host +1399 ServerName target = localServers.get(0); +1400 assignments.get(target).add(region); +1401 cluster.doAssignRegion(region, target); +1402 numRetainedAssigments++; +1403 } else { +1404 // multiple new servers in the cluster on this same host +1405 if (localServers.contains(oldServerName)) { +1406 assignments.get(oldServerName).add(region); +1407 cluster.doAssignRegion(region, oldServerName); +1408 } else { +1409 ServerName target = null; +1410 for (ServerName tmp: localServers) { +1411 if (tmp.getPort() == oldServerName.getPort()) { +1412 target = tmp; +1413 break; +1414 } +1415 } +1416 if (target == null) { +1417 target = randomAssignment(cluster, region, localServers); +1418 } +1419 assignments.get(target).add(region); +1420 } +1421 numRetainedAssigments++; 1422 } -1423 if (localServers.isEmpty()) { -1424 // No servers on the new cluster match up with this hostname, -1425 // assign randomly. -1426 ServerName randomServer = randomAssignment(cluster, region, servers); -1427 assignments.get(randomServer).add(region); -1428 numRandomAssignments++; -1429 if (oldServerName != null) oldHostsNoLongerPresent.add(oldServerName.getHostname()); -1430 } else if (localServers.size() == 1) { -1431 // the usual case - one new server on same host -1432 ServerName target = localServers.get(0); -1433 assignments.get(target).add(region); -1434 cluster.doAssignRegion(region, target); -1435 numRetainedAssigments++; -1436 } else { -1437 // multiple new servers in the cluster on this same host -1438 if (localServers.contains(oldServerName)) { -1439 assignments.get(oldServerName).add(region); -1440 cluster.doAssignRegion(region, oldServerName); -1441 } else { -1442 ServerName target = null; -1443 for (ServerName tmp: localServers) { -1444 if (tmp.getPort() == oldServerName.getPort()) { -1445 target = tmp; -1446 break; -1447 } -1448 } -1449 if (target == null) { -1450 target = randomAssignment(cluster, region, localServers); -1451 } -1452 assignments.get(target).add(region); -1453 } -1454 numRetainedAssigments++; -1455 } -1456 } -1457 -1458 String randomAssignMsg = ""; -1459 if (numRandomAssignments > 0) { -1460 randomAssignMsg = -1461 numRandomAssignments + " regions were assigned " -1462 + "to random hosts, since the old hosts for these regions are no " -1463 + "longer present in the cluster. These hosts were:\n " -1464 + Joiner.on("\n ").join(oldHostsNoLongerPresent); -1465 } -1466 -1467 LOG.info("Reassigned " + regions.size() + " regions. " + numRetainedAssigments -1468 + " retained the pre-restart assignment. " + randomAssignMsg); -1469 return assignments; -1470 } +1423 } +1424 +1425 String randomAssignMsg = ""; +1426 if (numRandomAssignments > 0) { +1427 randomAssignMsg = +1428 numRandomAssignments + " regions were assigned " +1429 + "to random hosts, since the old hosts for these regions are no " +1430 + "longer present in the cluster. These hosts were:\n " +1431 + Joiner.on("\n ").join(oldHostsNoLongerPresent); +1432 } +1433 +1434 LOG.info("Reassigned " + regions.size() + " regions. " + numRetainedAssigments +1435 + " retained the pre-restart assignment. " + randomAssignMsg); +1436 return assignments; +1437 } +1438 +1439 @Override +1440 public void initialize() throws HBaseIOException{ +1441 } +1442 +1443 @Override +1444 public void regionOnline(HRegionInfo regionInfo, ServerName sn) { +1445 } +1446 +1447 @Override +1448 public void regionOffline(HRegionInfo regionInfo) { +1449 } +1450 +1451 @Override +1452 public boolean isStopped() { +1453 return stopped; +1454 } +1455 +1456 @Override +1457 public void stop(String why) { +1458 LOG.info("Load Balancer stop requested: "+why); +1459 stopped = true; +1460 } +1461 +1462 /** +1463 * Used to assign a single region to a random server. +1464 */ +1465 private ServerName randomAssignment(Cluster cluster, HRegionInfo regionInfo, +1466 List<ServerName> servers) { +1467 int numServers = servers.size(); // servers is not null, numServers > 1 +1468 ServerName sn = null; +1469 final int maxIterations = numServers * 4; +1470 int iterations = 0; 1471 -1472 @Override -1473 public void initialize() throws HBaseIOException{ -1474 } -1475 -1476 @Override -1477 public void regionOnline(HRegionInfo regionInfo, ServerName sn) { -1478 } -1479 -1480 @Override -1481 public void regionOffline(HRegionInfo regionInfo) { -1482 } -1483 -1484 @Override -1485 public boolean isStopped() { -1486 return stopped; -1487 } -1488 -1489 @Override -1490 public void stop(String why) { -1491 LOG.info("Load Balancer stop requested: "+why); -1492 stopped = true; -1493 } -1494 -1495 /** -1496 * Used to assign a single region to a random server. -1497 */ -1498 private ServerName randomAssignment(Cluster cluster, HRegionInfo regionInfo, -1499 List<ServerName> servers) { -1500 int numServers = servers.size(); // servers is not null, numServers > 1 -1501 ServerName sn = null; -1502 final int maxIterations = numServers * 4; -1503 int iterations = 0; -1504 -1505 do { -1506 int i = RANDOM.nextInt(numServers); -1507 sn = servers.get(i); -1508 } while (cluster.wouldLowerAvailability(regionInfo, sn) -1509 && iterations++ < maxIterations); -1510 cluster.doAssignRegion(regionInfo, sn); -1511 return sn; +1472 do { +1473 int i = RANDOM.nextInt(numServers); +1474 sn = servers.get(i); +1475 } while (cluster.wouldLowerAvailability(regionInfo, sn) +1476 && iterations++ < maxIterations); +1477 cluster.doAssignRegion(regionInfo, sn); +1478 return sn; +1479 } +1480 +1481 /** +1482 * Round robin a list of regions to a list of servers +1483 */ +1484 private void roundRobinAssignment(Cluster cluster, List<HRegionInfo> regions, +1485 List<HRegionInfo> unassignedRegions, List<ServerName> servers, +1486 Map<ServerName, List<HRegionInfo>> assignments) { +1487 +1488 int numServers = servers.size(); +1489 int numRegions = regions.size(); +1490 int max = (int) Math.ceil((float) numRegions / numServers); +1491 int serverIdx = 0; +1492 if (numServers > 1) { +1493 serverIdx = RANDOM.nextInt(numServers); +1494 } +1495 int regionIdx = 0; +1496 +1497 for (int j = 0; j < numServers; j++) { +1498 ServerName server = servers.get((j + serverIdx) % numServers); +1499 List<HRegionInfo> serverRegions = new ArrayList<HRegionInfo>(max); +1500 for (int i = regionIdx; i < numRegions; i += numServers) { +1501 HRegionInfo region = regions.get(i % numRegions); +1502 if (cluster.wouldLowerAvailability(region, server)) { +1503 unassignedRegions.add(region); +1504 } else { +1505 serverRegions.add(region); +1506 cluster.doAssignRegion(region, server); +1507 } +1508 } +1509 assignments.put(server, serverRegions); +1510 regionIdx++; +1511 } 1512 } 1513 -1514 /** -1515 * Round robin a list of regions to a list of servers -1516 */ -1517 private void roundRobinAssignment(Cluster cluster, List<HRegionInfo> regions, -1518 List<HRegionInfo> unassignedRegions, List<ServerName> servers, -1519 Map<ServerName, List<HRegionInfo>> assignments) { -1520 -1521 int numServers = servers.size(); -1522 int numRegions = regions.size(); -1523 int max = (int) Math.ceil((float) numRegions / numServers); -1524 int serverIdx = 0; -1525 if (numServers > 1) { -1526 serverIdx = RANDOM.nextInt(numServers); -1527 } -1528 int regionIdx = 0; -1529 -1530 for (int j = 0; j < numServers; j++) { -1531 ServerName server = servers.get((j + serverIdx) % numServers); -1532 List<HRegionInfo> serverRegions = new ArrayList<HRegionInfo>(max); -1533 for (int i = regionIdx; i < numRegions; i += numServers) { -1534 HRegionInfo region = regions.get(i % numRegions); -1535 if (cluster.wouldLowerAvailability(region, server)) { -1536 unassignedRegions.add(region); -1537 } else { -1538 serverRegions.add(region); -1539 cluster.doAssignRegion(region, server); -1540 } -1541 } -1542 assignments.put(server, serverRegions); -1543 regionIdx++; -1544 } -1545 } -1546 -1547 protected Map<ServerName, List<HRegionInfo>> getRegionAssignmentsByServer( -1548 Collection<HRegionInfo> regions) { -1549 if (this.services != null && this.services.getAssignmentManager() != null) { -1550 return this.services.getAssignmentManager().getSnapShotOfAssignment(regions); -1551 } else { -1552 return new HashMap<ServerName, List<HRegionInfo>>(); -1553 } -1554 } -1555 -1556 @Override -1557 public void onConfigurationChange(Configuration conf) { -1558 } -1559} +1514 protected Map<ServerName, List<HRegionInfo>> getRegionAssignmentsByServer( +1515 Collection<HRegionInfo> regions) { +1516 if (this.services != null && this.services.getAssignmentManager() != null) { +1517 return this.services.getAssignmentManager().getSnapShotOfAssignment(regions); +1518 } else { +1519 return new HashMap<ServerName, List<HRegionInfo>>(); +1520 } +1521 } +1522 +1523 @Override +1524 public void onConfigurationChange(Configuration conf) { +1525 } +1526}