Return-Path: X-Original-To: apmail-hbase-commits-archive@www.apache.org Delivered-To: apmail-hbase-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 0574818E11 for ; Fri, 14 Aug 2015 17:07:22 +0000 (UTC) Received: (qmail 60514 invoked by uid 500); 14 Aug 2015 17:07:21 -0000 Delivered-To: apmail-hbase-commits-archive@hbase.apache.org Received: (qmail 60455 invoked by uid 500); 14 Aug 2015 17:07:21 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 60440 invoked by uid 99); 14 Aug 2015 17:07:21 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 14 Aug 2015 17:07:21 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 9FEE0E056D; Fri, 14 Aug 2015 17:07:21 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: apurtell@apache.org To: commits@hbase.apache.org Date: Fri, 14 Aug 2015 17:07:21 -0000 Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: [1/6] hbase git commit: HBASE-6721 RegionServer Group based Assignment (Francis Liu) Repository: hbase Updated Branches: refs/heads/hbase-6721-0.98 [created] 66e16163f http://git-wip-us.apache.org/repos/asf/hbase/blob/66e16163/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java index e40a32b..234cf4d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java @@ -26,10 +26,15 @@ import static org.junit.Assert.fail; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -38,6 +43,7 @@ import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.MiniHBaseCluster; import org.apache.hadoop.hbase.MiniHBaseCluster.MiniHBaseClusterRegionServer; @@ -149,7 +155,7 @@ public class TestAssignmentManagerOnCluster { TEST_UTIL.deleteTable(Bytes.toBytes(table)); } } - + // Simulate a scenario where the AssignCallable and SSH are trying to assign a region @Test (timeout=60000) public void testAssignRegionBySSH() throws Exception { @@ -179,15 +185,15 @@ public class TestAssignmentManagerOnCluster { TEST_UTIL.getHBaseCluster().killRegionServer(controlledServer); TEST_UTIL.getHBaseCluster().waitForRegionServerToStop(controlledServer, -1); AssignmentManager am = master.getAssignmentManager(); - + // Simulate the AssignCallable trying to assign the region. Have the region in OFFLINE state, - // but not in transition and the server is the dead 'controlledServer' + // but not in transition and the server is the dead 'controlledServer' regionStates.createRegionState(hri, State.OFFLINE, controlledServer); am.assign(hri, true, true); // Region should remain in OFFLINE and go to transition assertEquals(State.OFFLINE, regionStates.getRegionState(hri).getState()); assertTrue (regionStates.isRegionInTransition(hri)); - + master.enableSSH(true); am.waitForAssignment(hri); assertTrue (regionStates.getRegionState(hri).isOpened()); @@ -211,7 +217,7 @@ public class TestAssignmentManagerOnCluster { TEST_UTIL.getMiniHBaseCluster().getConf().setInt("hbase.assignment.maximum.attempts", 20); TEST_UTIL.getMiniHBaseCluster().stopMaster(0); TEST_UTIL.getMiniHBaseCluster().startMaster(); //restart the master so that conf take into affect - + ServerName deadServer = null; HMaster master = null; try { @@ -451,7 +457,7 @@ public class TestAssignmentManagerOnCluster { assertTrue(am.waitForAssignment(hri)); ServerName sn = am.getRegionStates().getRegionServerOfRegion(hri); TEST_UTIL.assertRegionOnServer(hri, sn, 6000); - + MyRegionObserver.preCloseEnabled.set(true); am.unassign(hri); RegionState state = am.getRegionStates().getRegionState(hri); @@ -566,6 +572,105 @@ public class TestAssignmentManagerOnCluster { } /** + * This tests round-robin assignment failed due to no bulkplan + */ + @Test (timeout=60000) + public void testRoundRobinAssignmentFailed() throws Exception { + String table = "testRoundRobinAssignmentFailed"; + try { + HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(table)); + desc.addFamily(new HColumnDescriptor(FAMILY)); + admin.createTable(desc); + + HTable meta = new HTable(conf, TableName.META_TABLE_NAME); + HRegionInfo hri = new HRegionInfo( + desc.getTableName(), Bytes.toBytes("A"), Bytes.toBytes("Z")); + MetaEditor.addRegionToMeta(meta, hri); + + MyLoadBalancer.controledRegion = hri.getEncodedName(); + + HMaster master = TEST_UTIL.getHBaseCluster().getMaster(); + AssignmentManager am = master.getAssignmentManager(); + // round-robin assignment but balancer cannot find a plan + // assignment should fail + am.assign(Arrays.asList(hri)); + + // if bulk assignment cannot update region state to online + // or failed_open this waits until timeout + assertFalse(am.waitForAssignment(hri)); + RegionState state = am.getRegionStates().getRegionState(hri); + assertEquals(RegionState.State.FAILED_OPEN, state.getState()); + // Failed to open since no plan, so it's on no server + assertNull(state.getServerName()); + + // try again with valid plan + MyLoadBalancer.controledRegion = null; + am.assign(Arrays.asList(hri)); + assertTrue(am.waitForAssignment(hri)); + + ServerName serverName = master.getAssignmentManager(). + getRegionStates().getRegionServerOfRegion(hri); + TEST_UTIL.assertRegionOnServer(hri, serverName, 200); + } finally { + MyLoadBalancer.controledRegion = null; + TEST_UTIL.deleteTable(Bytes.toBytes(table)); + } + } + + /** + * This tests retain assignment failed due to no bulkplan + */ + @Test (timeout=60000) + public void testRetainAssignmentFailed() throws Exception { + String table = "testRetainAssignmentFailed"; + try { + HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(table)); + desc.addFamily(new HColumnDescriptor(FAMILY)); + admin.createTable(desc); + + HTable meta = new HTable(conf, TableName.META_TABLE_NAME); + HRegionInfo hri = new HRegionInfo( + desc.getTableName(), Bytes.toBytes("A"), Bytes.toBytes("Z")); + MetaEditor.addRegionToMeta(meta, hri); + + MyLoadBalancer.controledRegion = hri.getEncodedName(); + + HMaster master = TEST_UTIL.getHBaseCluster().getMaster(); + AssignmentManager am = master.getAssignmentManager(); + + Map regions = new HashMap(); + ServerName dest = TEST_UTIL.getHBaseCluster().getRegionServer(0).getServerName(); + regions.put(hri, dest); + // retainAssignment but balancer cannot find a plan + // assignment should fail + am.assign(regions); + + // if retain assignment cannot update region state to online + // or failed_open this waits until timeout + assertFalse(am.waitForAssignment(hri)); + RegionState state = am.getRegionStates().getRegionState(hri); + assertEquals(RegionState.State.FAILED_OPEN, state.getState()); + // Failed to open since no plan, so it's on no server + assertNull(state.getServerName()); + + // try retainAssigment again with valid plan + MyLoadBalancer.controledRegion = null; + am.assign(regions); + assertTrue(am.waitForAssignment(hri)); + + ServerName serverName = master.getAssignmentManager(). + getRegionStates().getRegionServerOfRegion(hri); + TEST_UTIL.assertRegionOnServer(hri, serverName, 200); + + // it retains on same server as specified + assertEquals(serverName, dest); + } finally { + MyLoadBalancer.controledRegion = null; + TEST_UTIL.deleteTable(Bytes.toBytes(table)); + } + } + + /** * This tests region open failure which is not recoverable */ @Test (timeout=60000) @@ -646,7 +751,7 @@ public class TestAssignmentManagerOnCluster { Bytes.toBytes(rs.getServerName().getServerName())); am.waitForAssignment(HRegionInfo.FIRST_META_REGIONINFO); } - + am.regionOffline(hri); ZooKeeperWatcher zkw = TEST_UTIL.getHBaseCluster().getMaster().getZooKeeper(); am.getRegionStates().updateRegionState(hri, State.PENDING_OPEN, destServerName); @@ -787,7 +892,7 @@ public class TestAssignmentManagerOnCluster { List regions = new ArrayList(); regions.add(hri); am.assign(destServerName, regions); - + // let region open continue MyRegionObserver.postOpenEnabled.set(false); @@ -1017,6 +1122,31 @@ public class TestAssignmentManagerOnCluster { // For this region, if specified, always assign to nowhere static volatile String controledRegion = null; + + @Override + public Map> roundRobinAssignment( + List regions, List servers) { + if (regions.get(0).getEncodedName().equals(controledRegion)) { + Map> m = Maps.newHashMap(); + m.put(LoadBalancer.BOGUS_SERVER_NAME, regions); + return m; + } + return super.roundRobinAssignment(regions, servers); + } + + @Override + public Map> retainAssignment( + Map regions, List servers) { + for (HRegionInfo hri : regions.keySet()) { + if (hri.getEncodedName().equals(controledRegion)) { + Map> m = Maps.newHashMap(); + m.put(LoadBalancer.BOGUS_SERVER_NAME, Lists.newArrayList(regions.keySet())); + return m; + } + } + return super.retainAssignment(regions, servers); + } + @Override public ServerName randomAssignment(HRegionInfo regionInfo, List servers) { @@ -1047,7 +1177,7 @@ public class TestAssignmentManagerOnCluster { } } } - + public static class MyRegionServer extends MiniHBaseClusterRegionServer { static volatile ServerName abortedServer = null; static volatile boolean simulateRetry; http://git-wip-us.apache.org/repos/asf/hbase/blob/66e16163/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java index d39fb86..b829819 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java @@ -56,6 +56,8 @@ import org.apache.hadoop.hbase.client.HConnectionManager; import org.apache.hadoop.hbase.client.HConnectionTestingUtility; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.executor.ExecutorService; +import org.apache.hadoop.hbase.group.GroupAdmin; +import org.apache.hadoop.hbase.group.GroupAdminServer; import org.apache.hadoop.hbase.io.Reference; import org.apache.hadoop.hbase.master.CatalogJanitor.SplitParentFirstComparator; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; @@ -314,32 +316,29 @@ public class TestCatalogJanitor { @Override public void createNamespace(NamespaceDescriptor descriptor) throws IOException { - //To change body of implemented methods use File | Settings | File Templates. } @Override public void modifyNamespace(NamespaceDescriptor descriptor) throws IOException { - //To change body of implemented methods use File | Settings | File Templates. } @Override public void deleteNamespace(String name) throws IOException { - //To change body of implemented methods use File | Settings | File Templates. } @Override public NamespaceDescriptor getNamespaceDescriptor(String name) throws IOException { - return null; //To change body of implemented methods use File | Settings | File Templates. + return null; } @Override public List listNamespaceDescriptors() throws IOException { - return null; //To change body of implemented methods use File | Settings | File Templates. + return null; } @Override public List listTableDescriptorsByNamespace(String name) throws IOException { - return null; //To change body of implemented methods use File | Settings | File Templates. + return null; } @Override @@ -348,6 +347,16 @@ public class TestCatalogJanitor { } @Override + public LoadBalancer getLoadBalancer() { + return null; + } + + @Override + public GroupAdminServer getGroupAdminServer() { + return null; + } + + @Override public void deleteTable(TableName tableName) throws IOException { } @Override http://git-wip-us.apache.org/repos/asf/hbase/blob/66e16163/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestGroupBasedLoadBalancer.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestGroupBasedLoadBalancer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestGroupBasedLoadBalancer.java new file mode 100644 index 0000000..72bf887 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestGroupBasedLoadBalancer.java @@ -0,0 +1,588 @@ +/** + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Lists; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.HostPort; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableDescriptors; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.group.GroupBasedLoadBalancer; +import org.apache.hadoop.hbase.group.GroupInfo; +import org.apache.hadoop.hbase.group.GroupInfoManager; +import org.apache.hadoop.hbase.master.AssignmentManager; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.MasterServices; +import org.apache.hadoop.hbase.master.RegionPlan; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.security.SecureRandom; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +//TODO use stochastic based load balancer instead +@Category(SmallTests.class) +public class TestGroupBasedLoadBalancer { + + private static final Log LOG = LogFactory.getLog(TestGroupBasedLoadBalancer.class); + private static GroupBasedLoadBalancer loadBalancer; + private static SecureRandom rand; + + static String[] groups = new String[] { GroupInfo.DEFAULT_GROUP, "dg2", "dg3", + "dg4" }; + static TableName[] tables = + new TableName[] { TableName.valueOf("dt1"), + TableName.valueOf("dt2"), + TableName.valueOf("dt3"), + TableName.valueOf("dt4")}; + static List servers; + static Map groupMap; + static Map tableMap; + static List tableDescs; + int[] regionAssignment = new int[] { 2, 5, 7, 10, 4, 3, 1 }; + static int regionId = 0; + + @BeforeClass + public static void beforeAllTests() throws Exception { + rand = new SecureRandom(); + servers = generateServers(7); + groupMap = constructGroupInfo(servers, groups); + tableMap = new HashMap(); + tableDescs = constructTableDesc(); + Configuration conf = HBaseConfiguration.create(); + conf.set("hbase.regions.slop", "0"); + conf.set("hbase.group.grouploadbalancer.class", SimpleLoadBalancer.class.getCanonicalName()); + loadBalancer = new GroupBasedLoadBalancer(getMockedGroupInfoManager()); + loadBalancer.setMasterServices(getMockedMaster()); + loadBalancer.setConf(conf); + loadBalancer.initialize(); + } + + /** + * Test the load balancing algorithm. + * + * Invariant is that all servers of the group should be hosting either floor(average) or + * ceiling(average) + * + * @throws Exception + */ + @Test + public void testBalanceCluster() throws Exception { + Map> servers = mockClusterServers(); + ArrayListMultimap list = convertToGroupBasedMap(servers); + LOG.info("Mock Cluster : " + printStats(list)); + List plans = loadBalancer.balanceCluster(servers); + ArrayListMultimap balancedCluster = reconcile( + list, plans); + LOG.info("Mock Balance : " + printStats(balancedCluster)); + assertClusterAsBalanced(balancedCluster); + } + + /** + * Invariant is that all servers of a group have load between floor(avg) and + * ceiling(avg) number of regions. + */ + private void assertClusterAsBalanced( + ArrayListMultimap groupLoadMap) { + for (String gName : groupLoadMap.keySet()) { + List groupLoad = groupLoadMap.get(gName); + int numServers = groupLoad.size(); + int numRegions = 0; + int maxRegions = 0; + int minRegions = Integer.MAX_VALUE; + for (ServerAndLoad server : groupLoad) { + int nr = server.getLoad(); + if (nr > maxRegions) { + maxRegions = nr; + } + if (nr < minRegions) { + minRegions = nr; + } + numRegions += nr; + } + if (maxRegions - minRegions < 2) { + // less than 2 between max and min, can't balance + return; + } + int min = numRegions / numServers; + int max = numRegions % numServers == 0 ? min : min + 1; + + for (ServerAndLoad server : groupLoad) { + assertTrue(server.getLoad() <= max); + assertTrue(server.getLoad() >= min); + } + } + } + + /** + * Tests immediate assignment. + * + * Invariant is that all regions have an assignment. + * + * @throws Exception + */ + @Test + public void testImmediateAssignment() throws Exception { + List regions = randomRegions(20); + Map assignments = loadBalancer + .immediateAssignment(regions, servers); + assertImmediateAssignment(regions, servers, assignments); + } + + /** + * All regions have an assignment. + * + * @param regions + * @param servers + * @param assignments + * @throws java.io.IOException + * @throws java.io.FileNotFoundException + */ + private void assertImmediateAssignment(List regions, + List servers, Map assignments) + throws FileNotFoundException, IOException { + for (HRegionInfo region : regions) { + assertTrue(assignments.containsKey(region)); + ServerName server = assignments.get(region); + TableName tableName = region.getTable(); + + String groupName = + loadBalancer.getGroupInfoManager().getGroupOfTable(tableName); + assertTrue(StringUtils.isNotEmpty(groupName)); + GroupInfo gInfo = getMockedGroupInfoManager().getGroup(groupName); + assertTrue("Region is not correctly assigned to group servers.", + gInfo.containsServer(server.getHostPort())); + } + } + + /** + * Tests the bulk assignment used during cluster startup. + * + * Round-robin. Should yield a balanced cluster so same invariant as the + * load balancer holds, all servers holding either floor(avg) or + * ceiling(avg). + * + * @throws Exception + */ + @Test + public void testBulkAssignment() throws Exception { + List regions = randomRegions(25); + Map> assignments = loadBalancer + .roundRobinAssignment(regions, servers); + //test empty region/servers scenario + //this should not throw an NPE + loadBalancer.roundRobinAssignment(regions, + Collections.EMPTY_LIST); + //test regular scenario + assertTrue(assignments.keySet().size() == servers.size()); + for (ServerName sn : assignments.keySet()) { + List regionAssigned = assignments.get(sn); + for (HRegionInfo region : regionAssigned) { + TableName tableName = region.getTable(); + String groupName = + getMockedGroupInfoManager().getGroupOfTable(tableName); + assertTrue(StringUtils.isNotEmpty(groupName)); + GroupInfo gInfo = getMockedGroupInfoManager().getGroup( + groupName); + assertTrue( + "Region is not correctly assigned to group servers.", + gInfo.containsServer(sn.getHostPort())); + } + } + ArrayListMultimap loadMap = convertToGroupBasedMap(assignments); + assertClusterAsBalanced(loadMap); + } + + /** + * Test the cluster startup bulk assignment which attempts to retain + * assignment info. + * + * @throws Exception + */ + @Test + public void testRetainAssignment() throws Exception { + // Test simple case where all same servers are there + Map> currentAssignments = mockClusterServers(); + Map inputForTest = new HashMap(); + for (ServerName sn : currentAssignments.keySet()) { + for (HRegionInfo region : currentAssignments.get(sn)) { + inputForTest.put(region, sn); + } + } + //verify region->null server assignment is handled + inputForTest.put(randomRegions(1).get(0), null); + Map> newAssignment = loadBalancer + .retainAssignment(inputForTest, servers); + assertRetainedAssignment(inputForTest, servers, newAssignment); + } + + /** + * Asserts a valid retained assignment plan. + *

+ * Must meet the following conditions: + *

    + *
  • Every input region has an assignment, and to an online server + *
  • If a region had an existing assignment to a server with the same + * address a a currently online server, it will be assigned to it + *
+ * + * @param existing + * @param assignment + * @throws java.io.IOException + * @throws java.io.FileNotFoundException + */ + private void assertRetainedAssignment( + Map existing, List servers, + Map> assignment) + throws FileNotFoundException, IOException { + // Verify condition 1, every region assigned, and to online server + Set onlineServerSet = new TreeSet(servers); + Set assignedRegions = new TreeSet(); + for (Map.Entry> a : assignment.entrySet()) { + assertTrue( + "Region assigned to server that was not listed as online", + onlineServerSet.contains(a.getKey())); + for (HRegionInfo r : a.getValue()) + assignedRegions.add(r); + } + assertEquals(existing.size(), assignedRegions.size()); + + // Verify condition 2, every region must be assigned to correct server. + Set onlineHostNames = new TreeSet(); + for (ServerName s : servers) { + onlineHostNames.add(s.getHostname()); + } + + for (Map.Entry> a : assignment.entrySet()) { + ServerName currentServer = a.getKey(); + for (HRegionInfo r : a.getValue()) { + ServerName oldAssignedServer = existing.get(r); + TableName tableName = r.getTable(); + String groupName = + getMockedGroupInfoManager().getGroupOfTable(tableName); + assertTrue(StringUtils.isNotEmpty(groupName)); + GroupInfo gInfo = getMockedGroupInfoManager().getGroup( + groupName); + assertTrue( + "Region is not correctly assigned to group servers.", + gInfo.containsServer(currentServer.getHostPort())); + if (oldAssignedServer != null + && onlineHostNames.contains(oldAssignedServer + .getHostname())) { + // this region was previously assigned somewhere, and that + // host is still around, then the host must have been is a + // different group. + if (!oldAssignedServer.getHostPort().equals(currentServer.getHostPort())) { + assertFalse(gInfo.containsServer(oldAssignedServer.getHostPort())); + } + } + } + } + } + + private String printStats( + ArrayListMultimap groupBasedLoad) { + StringBuffer sb = new StringBuffer(); + sb.append("\n"); + for (String groupName : groupBasedLoad.keySet()) { + sb.append("Stats for group: " + groupName); + sb.append("\n"); + sb.append(groupMap.get(groupName).getServers()); + sb.append("\n"); + List groupLoad = groupBasedLoad.get(groupName); + int numServers = groupLoad.size(); + int totalRegions = 0; + sb.append("Per Server Load: \n"); + for (ServerAndLoad sLoad : groupLoad) { + sb.append("Server :" + sLoad.getServerName() + " Load : " + + sLoad.getLoad() + "\n"); + totalRegions += sLoad.getLoad(); + } + sb.append(" Group Statistics : \n"); + float average = (float) totalRegions / numServers; + int max = (int) Math.ceil(average); + int min = (int) Math.floor(average); + sb.append("[srvr=" + numServers + " rgns=" + totalRegions + " avg=" + + average + " max=" + max + " min=" + min + "]"); + sb.append("\n"); + sb.append("==============================="); + sb.append("\n"); + } + return sb.toString(); + } + + private ArrayListMultimap convertToGroupBasedMap( + final Map> serversMap) throws IOException { + ArrayListMultimap loadMap = ArrayListMultimap + .create(); + for (GroupInfo gInfo : getMockedGroupInfoManager().listGroups()) { + Set groupServers = gInfo.getServers(); + for (HostPort hostPort : groupServers) { + ServerName actual = null; + for(ServerName entry: servers) { + if(entry.getHostPort().equals(hostPort)) { + actual = entry; + break; + } + } + List regions = serversMap.get(actual); + assertTrue("No load for " + actual, regions != null); + loadMap.put(gInfo.getName(), + new ServerAndLoad(actual, regions.size())); + } + } + return loadMap; + } + + private ArrayListMultimap reconcile( + ArrayListMultimap previousLoad, + List plans) { + ArrayListMultimap result = ArrayListMultimap + .create(); + result.putAll(previousLoad); + if (plans != null) { + for (RegionPlan plan : plans) { + ServerName source = plan.getSource(); + updateLoad(result, source, -1); + ServerName destination = plan.getDestination(); + updateLoad(result, destination, +1); + } + } + return result; + } + + private void updateLoad( + ArrayListMultimap previousLoad, + final ServerName sn, final int diff) { + for (String groupName : previousLoad.keySet()) { + ServerAndLoad newSAL = null; + ServerAndLoad oldSAL = null; + for (ServerAndLoad sal : previousLoad.get(groupName)) { + if (ServerName.isSameHostnameAndPort(sn, sal.getServerName())) { + oldSAL = sal; + newSAL = new ServerAndLoad(sn, sal.getLoad() + diff); + break; + } + } + if (newSAL != null) { + previousLoad.remove(groupName, oldSAL); + previousLoad.put(groupName, newSAL); + break; + } + } + } + + private Map> mockClusterServers() throws IOException { + assertTrue(servers.size() == regionAssignment.length); + Map> assignment = new TreeMap>(); + for (int i = 0; i < servers.size(); i++) { + int numRegions = regionAssignment[i]; + List regions = assignedRegions(numRegions, servers.get(i)); + assignment.put(servers.get(i), regions); + } + return assignment; + } + + /** + * Generate a list of regions evenly distributed between the tables. + * + * @param numRegions The number of regions to be generated. + * @return List of HRegionInfo. + */ + private List randomRegions(int numRegions) { + List regions = new ArrayList(numRegions); + byte[] start = new byte[16]; + byte[] end = new byte[16]; + rand.nextBytes(start); + rand.nextBytes(end); + int regionIdx = rand.nextInt(tables.length); + for (int i = 0; i < numRegions; i++) { + Bytes.putInt(start, 0, numRegions << 1); + Bytes.putInt(end, 0, (numRegions << 1) + 1); + int tableIndex = (i + regionIdx) % tables.length; + HRegionInfo hri = new HRegionInfo( + tables[tableIndex], start, end, false, regionId++); + regions.add(hri); + } + return regions; + } + + /** + * Generate assigned regions to a given server using group information. + * + * @param numRegions the num regions to generate + * @param sn the servername + * @return the list of regions + * @throws java.io.IOException Signals that an I/O exception has occurred. + */ + private List assignedRegions(int numRegions, ServerName sn) throws IOException { + List regions = new ArrayList(numRegions); + byte[] start = new byte[16]; + byte[] end = new byte[16]; + Bytes.putInt(start, 0, numRegions << 1); + Bytes.putInt(end, 0, (numRegions << 1) + 1); + for (int i = 0; i < numRegions; i++) { + TableName tableName = getTableName(sn); + HRegionInfo hri = new HRegionInfo( + tableName, start, end, false, + regionId++); + regions.add(hri); + } + return regions; + } + + private static List generateServers(int numServers) { + List servers = new ArrayList(numServers); + for (int i = 0; i < numServers; i++) { + String host = "server" + rand.nextInt(100000); + int port = rand.nextInt(60000); + servers.add(ServerName.valueOf(host, port, -1)); + } + return servers; + } + + /** + * Construct group info, with each group having at least one server. + * + * @param servers the servers + * @param groups the groups + * @return the map + */ + private static Map constructGroupInfo( + List servers, String[] groups) { + assertTrue(servers != null); + assertTrue(servers.size() >= groups.length); + int index = 0; + Map groupMap = new HashMap(); + for (String grpName : groups) { + GroupInfo groupInfo = new GroupInfo(grpName); + groupInfo.addServer(servers.get(index).getHostPort()); + groupMap.put(grpName, groupInfo); + index++; + } + while (index < servers.size()) { + int grpIndex = rand.nextInt(groups.length); + groupMap.get(groups[grpIndex]).addServer( + servers.get(index).getHostPort()); + index++; + } + return groupMap; + } + + /** + * Construct table descriptors evenly distributed between the groups. + * + * @return the list + */ + private static List constructTableDesc() { + List tds = Lists.newArrayList(); + int index = rand.nextInt(groups.length); + for (int i = 0; i < tables.length; i++) { + HTableDescriptor htd = new HTableDescriptor(tables[i]); + int grpIndex = (i + index) % groups.length ; + String groupName = groups[grpIndex]; + tableMap.put(tables[i], groupName); + tds.add(htd); + } + return tds; + } + + private static MasterServices getMockedMaster() throws IOException { + TableDescriptors tds = Mockito.mock(TableDescriptors.class); + Mockito.when(tds.get(tables[0])).thenReturn(tableDescs.get(0)); + Mockito.when(tds.get(tables[1])).thenReturn(tableDescs.get(1)); + Mockito.when(tds.get(tables[2])).thenReturn(tableDescs.get(2)); + Mockito.when(tds.get(tables[3])).thenReturn(tableDescs.get(3)); + MasterServices services = Mockito.mock(HMaster.class); + Mockito.when(services.getTableDescriptors()).thenReturn(tds); + AssignmentManager am = Mockito.mock(AssignmentManager.class); + Mockito.when(services.getAssignmentManager()).thenReturn(am); + return services; + } + + private static GroupInfoManager getMockedGroupInfoManager() throws IOException { + GroupInfoManager gm = Mockito.mock(GroupInfoManager.class); + Mockito.when(gm.getGroup(groups[0])).thenReturn( + groupMap.get(groups[0])); + Mockito.when(gm.getGroup(groups[1])).thenReturn( + groupMap.get(groups[1])); + Mockito.when(gm.getGroup(groups[2])).thenReturn( + groupMap.get(groups[2])); + Mockito.when(gm.getGroup(groups[3])).thenReturn( + groupMap.get(groups[3])); + Mockito.when(gm.listGroups()).thenReturn( + Lists.newLinkedList(groupMap.values())); + Mockito.when(gm.isOnline()).thenReturn(true); + Mockito.when(gm.getGroupOfTable(Mockito.any(TableName.class))) + .thenAnswer(new Answer() { + @Override + public String answer(InvocationOnMock invocation) throws Throwable { + return tableMap.get(invocation.getArguments()[0]); + } + }); + return gm; + } + + private TableName getTableName(ServerName sn) throws IOException { + TableName tableName = null; + GroupInfoManager gm = getMockedGroupInfoManager(); + GroupInfo groupOfServer = null; + for(GroupInfo gInfo : gm.listGroups()){ + if(gInfo.containsServer(sn.getHostPort())){ + groupOfServer = gInfo; + break; + } + } + + for(HTableDescriptor desc : tableDescs){ + if(gm.getGroupOfTable(desc.getTableName()).endsWith(groupOfServer.getName())){ + tableName = desc.getTableName(); + } + } + return tableName; + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/66e16163/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java index 8f4d8d7..e521424 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java @@ -2488,4 +2488,79 @@ public class TestAccessController extends SecureTestUtil { verifyDenied(replicateLogEntriesAction, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_OWNER, USER_GROUP_READ, USER_GROUP_ADMIN, USER_GROUP_CREATE); } + + @Test + public void testMoveServers() throws Exception { + AccessTestAction action1 = new AccessTestAction() { + @Override + public Object run() throws Exception { + ACCESS_CONTROLLER.preMoveServers(ObserverContext.createAndPrepare(CP_ENV, null), + null, null); + return null; + } + }; + + verifyAllowed(action1, SUPERUSER, USER_ADMIN); + verifyDenied(action1, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_OWNER); + } + + @Test + public void testMoveTables() throws Exception { + AccessTestAction action1 = new AccessTestAction() { + @Override + public Object run() throws Exception { + ACCESS_CONTROLLER.preMoveTables(ObserverContext.createAndPrepare(CP_ENV, null), + null, null); + return null; + } + }; + + verifyAllowed(action1, SUPERUSER, USER_ADMIN); + verifyDenied(action1, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_OWNER); + } + + @Test + public void testAddGroup() throws Exception { + AccessTestAction action1 = new AccessTestAction() { + @Override + public Object run() throws Exception { + ACCESS_CONTROLLER.preAddGroup(ObserverContext.createAndPrepare(CP_ENV, null), + null); + return null; + } + }; + + verifyAllowed(action1, SUPERUSER, USER_ADMIN); + verifyDenied(action1, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_OWNER); + } + + @Test + public void testRemoveGroup() throws Exception { + AccessTestAction action1 = new AccessTestAction() { + @Override + public Object run() throws Exception { + ACCESS_CONTROLLER.preRemoveGroup(ObserverContext.createAndPrepare(CP_ENV, null), + null); + return null; + } + }; + + verifyAllowed(action1, SUPERUSER, USER_ADMIN); + verifyDenied(action1, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_OWNER); + } + + @Test + public void testBalanceGroup() throws Exception { + AccessTestAction action1 = new AccessTestAction() { + @Override + public Object run() throws Exception { + ACCESS_CONTROLLER.preBalanceGroup(ObserverContext.createAndPrepare(CP_ENV, null), + null); + return null; + } + }; + + verifyAllowed(action1, SUPERUSER, USER_ADMIN); + verifyDenied(action1, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_OWNER); + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/66e16163/hbase-shell/src/main/ruby/hbase.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/main/ruby/hbase.rb b/hbase-shell/src/main/ruby/hbase.rb index 70ab8fe..b56b93f 100644 --- a/hbase-shell/src/main/ruby/hbase.rb +++ b/hbase-shell/src/main/ruby/hbase.rb @@ -89,3 +89,4 @@ require 'hbase/table' require 'hbase/replication_admin' require 'hbase/security' require 'hbase/visibility_labels' +require 'hbase/group_admin' http://git-wip-us.apache.org/repos/asf/hbase/blob/66e16163/hbase-shell/src/main/ruby/hbase/group_admin.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/main/ruby/hbase/group_admin.rb b/hbase-shell/src/main/ruby/hbase/group_admin.rb new file mode 100644 index 0000000..bb4cefe --- /dev/null +++ b/hbase-shell/src/main/ruby/hbase/group_admin.rb @@ -0,0 +1,121 @@ +# +# Copyright The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include Java +java_import org.apache.hadoop.hbase.util.Pair + +# Wrapper for org.apache.hadoop.hbase.group.GroupAdminClient +# Which is an API to manage region server groups + +module Hbase + class GroupAdmin + include HBaseConstants + + def initialize(configuration, formatter) + @admin = org.apache.hadoop.hbase.group.GroupAdminClient.new(configuration) + @conf = configuration + @formatter = formatter + end + + #---------------------------------------------------------------------------------------------- + # Returns a list of groups in hbase + def listGroups + @admin.listGroups.map { |g| g.getName } + end + #---------------------------------------------------------------------------------------------- + # get a group's information + def getGroup(group_name) + group = @admin.getGroupInfo(group_name) + res = {} + if block_given? + yield("Servers:") + else + res += v + end + group.getServers.each do |v| + if block_given? + yield(v.toString) + else + res += v.toString + end + end + if block_given? + yield("Tables:") + else + res += v + end + group.getTables.each do |v| + if block_given? + yield(v.toString) + else + res += v.toString + end + end + end + #---------------------------------------------------------------------------------------------- + # add a group + def addGroup(group_name) + @admin.addGroup(group_name) + end + #---------------------------------------------------------------------------------------------- + # remove a group + def removeGroup(group_name) + @admin.removeGroup(group_name) + end + #---------------------------------------------------------------------------------------------- + # balance a group + def balanceGroup(group_name) + @admin.balanceGroup(group_name) + end + #---------------------------------------------------------------------------------------------- + # move server to a group + def moveServers(dest, *args) + servers = java.util.HashSet.new() + args[0].each do |s| + servers.add(org.apache.hadoop.hbase.HostPort.valueOf(s)) + end + @admin.moveServers(servers, dest) + end + #---------------------------------------------------------------------------------------------- + # move server to a group + def moveTables(dest, *args) + tables = java.util.HashSet.new(); + args[0].each do |s| + tables.add(org.apache.hadoop.hbase.TableName.valueOf(s)) + end + @admin.moveTables(tables,dest) + end + #---------------------------------------------------------------------------------------------- + # get group of server + def getGroupOfServer(server) + @admin.getGroupOfServer(org.apache.hadoop.hbase.HostPort.valueOf(server)) + end + #---------------------------------------------------------------------------------------------- + # get group of server + def getGroupOfTable(table) + @admin.getGroupInfoOfTable(org.apache.hadoop.hbase.TableName.valueOf(table)) + end + #---------------------------------------------------------------------------------------------- + # get list tables of groups + def listTablesOfGroup(group_name) + @admin.listTablesOfGroup(group_name) + end + end +end http://git-wip-us.apache.org/repos/asf/hbase/blob/66e16163/hbase-shell/src/main/ruby/hbase/hbase.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/main/ruby/hbase/hbase.rb b/hbase-shell/src/main/ruby/hbase/hbase.rb index e75535e..458176f 100644 --- a/hbase-shell/src/main/ruby/hbase/hbase.rb +++ b/hbase-shell/src/main/ruby/hbase/hbase.rb @@ -44,6 +44,10 @@ module Hbase ::Hbase::Admin.new(configuration, formatter) end + def group_admin(formatter) + ::Hbase::GroupAdmin.new(configuration, formatter) + end + # Create new one each time def table(table, shell) ::Hbase::Table.new(configuration, table, shell) http://git-wip-us.apache.org/repos/asf/hbase/blob/66e16163/hbase-shell/src/main/ruby/shell.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/main/ruby/shell.rb b/hbase-shell/src/main/ruby/shell.rb index f56499c..c4500af 100644 --- a/hbase-shell/src/main/ruby/shell.rb +++ b/hbase-shell/src/main/ruby/shell.rb @@ -87,6 +87,10 @@ module Shell @hbase_admin ||= hbase.admin(formatter) end + def group_admin + @group_admin ||= hbase.group_admin(formatter) + end + def hbase_table(name) hbase.table(name, self) end @@ -386,3 +390,20 @@ Shell.load_command_group( set_visibility ] ) + +Shell.load_command_group( + 'group', + :full_name => 'Groups', + :comment => "NOTE: Above commands are only applicable if running with the Groups setup", + :commands => %w[ + list_groups + get_group + add_group + remove_group + balance_group + move_group_servers + move_group_tables + get_server_group + get_table_group + ] +) http://git-wip-us.apache.org/repos/asf/hbase/blob/66e16163/hbase-shell/src/main/ruby/shell/commands.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/main/ruby/shell/commands.rb b/hbase-shell/src/main/ruby/shell/commands.rb index 1b079fb..41538b9 100644 --- a/hbase-shell/src/main/ruby/shell/commands.rb +++ b/hbase-shell/src/main/ruby/shell/commands.rb @@ -54,6 +54,10 @@ module Shell @shell.hbase_admin end + def group_admin + @shell.group_admin + end + def table(name) @shell.hbase_table(name) end http://git-wip-us.apache.org/repos/asf/hbase/blob/66e16163/hbase-shell/src/main/ruby/shell/commands/add_group.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/main/ruby/shell/commands/add_group.rb b/hbase-shell/src/main/ruby/shell/commands/add_group.rb new file mode 100644 index 0000000..7f91ee5 --- /dev/null +++ b/hbase-shell/src/main/ruby/shell/commands/add_group.rb @@ -0,0 +1,39 @@ +# +# Copyright The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class AddGroup < Command + def help + return <<-EOF +Create a new region server group. + +Example: + + hbase> add_group 'my_group' +EOF + end + + def command(group_name) + group_admin.addGroup(group_name) + end + end + end +end http://git-wip-us.apache.org/repos/asf/hbase/blob/66e16163/hbase-shell/src/main/ruby/shell/commands/balance_group.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/main/ruby/shell/commands/balance_group.rb b/hbase-shell/src/main/ruby/shell/commands/balance_group.rb new file mode 100644 index 0000000..4c59f63 --- /dev/null +++ b/hbase-shell/src/main/ruby/shell/commands/balance_group.rb @@ -0,0 +1,37 @@ +# +# Copyright The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class BalanceGroup < Command + def help + return <<-EOF +Balance a region server group + + hbase> group_balance 'my_group' +EOF + end + + def command(group_name) + group_admin.balanceGroup(group_name) + end + end + end +end http://git-wip-us.apache.org/repos/asf/hbase/blob/66e16163/hbase-shell/src/main/ruby/shell/commands/get_group.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/main/ruby/shell/commands/get_group.rb b/hbase-shell/src/main/ruby/shell/commands/get_group.rb new file mode 100644 index 0000000..5ed8226 --- /dev/null +++ b/hbase-shell/src/main/ruby/shell/commands/get_group.rb @@ -0,0 +1,44 @@ +# +# Copyright The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class GetGroup < Command + def help + return <<-EOF +Get a region server group's information. + +Example: + + hbase> get_group 'default' +EOF + end + + def command(group_name) + now = Time.now + formatter.header([ "GROUP INFORMATION" ]) + group_admin.getGroup(group_name) do |s| + formatter.row([ s ]) + end + formatter.footer(now) + end + end + end +end http://git-wip-us.apache.org/repos/asf/hbase/blob/66e16163/hbase-shell/src/main/ruby/shell/commands/get_server_group.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/main/ruby/shell/commands/get_server_group.rb b/hbase-shell/src/main/ruby/shell/commands/get_server_group.rb new file mode 100644 index 0000000..c78d4d2 --- /dev/null +++ b/hbase-shell/src/main/ruby/shell/commands/get_server_group.rb @@ -0,0 +1,40 @@ +# +# Copyright The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class GetServerGroup < Command + def help + return <<-EOF +Get the group name the given region server is a member of. + + hbase> get_server_group 'server1:port1' +EOF + end + + def command(server) + now = Time.now + groupName = group_admin.getGroupOfServer(server).getName + formatter.row([ groupName ]) + formatter.footer(now,1) + end + end + end +end http://git-wip-us.apache.org/repos/asf/hbase/blob/66e16163/hbase-shell/src/main/ruby/shell/commands/get_table_group.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/main/ruby/shell/commands/get_table_group.rb b/hbase-shell/src/main/ruby/shell/commands/get_table_group.rb new file mode 100644 index 0000000..dd8766d --- /dev/null +++ b/hbase-shell/src/main/ruby/shell/commands/get_table_group.rb @@ -0,0 +1,41 @@ +# +# Copyright The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class GetTableGroup < Command + def help + return <<-EOF +Get the group name the given table is a member of. + + hbase> get_table_group 'myTable' +EOF + end + + def command(table) + now = Time.now + groupName = + group_admin.getGroupOfTable(table).getName + formatter.row([ groupName ]) + formatter.footer(now,1) + end + end + end +end http://git-wip-us.apache.org/repos/asf/hbase/blob/66e16163/hbase-shell/src/main/ruby/shell/commands/list_groups.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/main/ruby/shell/commands/list_groups.rb b/hbase-shell/src/main/ruby/shell/commands/list_groups.rb new file mode 100644 index 0000000..2e7dd08 --- /dev/null +++ b/hbase-shell/src/main/ruby/shell/commands/list_groups.rb @@ -0,0 +1,50 @@ +# +# Copyright The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ListGroups < Command + def help + return <<-EOF +List all region server groups. Optional regular expression parameter could +be used to filter the output. + +Example: + + hbase> list_groups + hbase> list_groups 'abc.*' +EOF + end + + def command(regex = ".*") + now = Time.now + formatter.header([ "GROUPS" ]) + + regex = /#{regex}/ unless regex.is_a?(Regexp) + list = group_admin.listGroups.grep(regex) + list.each do |group| + formatter.row([ group ]) + end + + formatter.footer(now, list.size) + end + end + end +end http://git-wip-us.apache.org/repos/asf/hbase/blob/66e16163/hbase-shell/src/main/ruby/shell/commands/move_group_servers.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/main/ruby/shell/commands/move_group_servers.rb b/hbase-shell/src/main/ruby/shell/commands/move_group_servers.rb new file mode 100644 index 0000000..5e5c850 --- /dev/null +++ b/hbase-shell/src/main/ruby/shell/commands/move_group_servers.rb @@ -0,0 +1,37 @@ +# +# Copyright The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class MoveGroupServers < Command + def help + return <<-EOF +Reassign a region server from one group to another. + + hbase> move_group_servers 'dest',['server1:port','server2:port'] +EOF + end + + def command(dest, *servers) + group_admin.moveServers(dest, *servers) + end + end + end +end http://git-wip-us.apache.org/repos/asf/hbase/blob/66e16163/hbase-shell/src/main/ruby/shell/commands/move_group_tables.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/main/ruby/shell/commands/move_group_tables.rb b/hbase-shell/src/main/ruby/shell/commands/move_group_tables.rb new file mode 100644 index 0000000..f495f2c --- /dev/null +++ b/hbase-shell/src/main/ruby/shell/commands/move_group_tables.rb @@ -0,0 +1,37 @@ +# +# Copyright The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class MoveGroupTables < Command + def help + return <<-EOF +Reassign tables from one group to another. + + hbase> move_group_tables 'dest',['table1','table2'] +EOF + end + + def command(dest, *servers) + group_admin.moveTables(dest, *servers) + end + end + end +end http://git-wip-us.apache.org/repos/asf/hbase/blob/66e16163/hbase-shell/src/main/ruby/shell/commands/remove_group.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/main/ruby/shell/commands/remove_group.rb b/hbase-shell/src/main/ruby/shell/commands/remove_group.rb new file mode 100644 index 0000000..66863a4 --- /dev/null +++ b/hbase-shell/src/main/ruby/shell/commands/remove_group.rb @@ -0,0 +1,37 @@ +# +# Copyright The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class RemoveGroup < Command + def help + return <<-EOF +Remove a group. + + hbase> remove_group 'my_group' +EOF + end + + def command(group_name) + group_admin.removeGroup(group_name) + end + end + end +end