hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From apurt...@apache.org
Subject [07/14] hbase git commit: HBASE-15631 Backport Regionserver Groups (HBASE-6721) to branch-1 (Francis Liu and Andrew Purtell)
Date Tue, 24 Oct 2017 01:02:58 GMT
HBASE-15631 Backport Regionserver Groups (HBASE-6721) to branch-1 (Francis Liu and Andrew Purtell)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c3200076
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c3200076
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c3200076

Branch: refs/heads/branch-1.4
Commit: c320007638fe57f858aeac974cdd6bbd6b9dd5eb
Parents: 737b5a5
Author: Andrew Purtell <apurtell@apache.org>
Authored: Mon Oct 23 14:15:06 2017 -0700
Committer: Andrew Purtell <apurtell@apache.org>
Committed: Mon Oct 23 17:10:23 2017 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/hbase/ServerName.java     |    25 +-
 .../org/apache/hadoop/hbase/net/Address.java    |    89 +
 hbase-it/pom.xml                                |    41 +
 .../hbase/rsgroup/IntegrationTestRSGroup.java   |    99 +
 hbase-protocol/pom.xml                          |     2 +
 .../hbase/protobuf/generated/ClientProtos.java  |    16 +-
 .../protobuf/generated/RSGroupAdminProtos.java  | 13571 +++++++++++++++++
 .../hbase/protobuf/generated/RSGroupProtos.java |  1332 ++
 hbase-protocol/src/main/protobuf/RSGroup.proto  |    35 +
 .../src/main/protobuf/RSGroupAdmin.proto        |   149 +
 hbase-rsgroup/pom.xml                           |   278 +
 .../hadoop/hbase/rsgroup/RSGroupAdmin.java      |    92 +
 .../hbase/rsgroup/RSGroupAdminClient.java       |   212 +
 .../hbase/rsgroup/RSGroupAdminEndpoint.java     |  1049 ++
 .../hbase/rsgroup/RSGroupAdminServer.java       |   516 +
 .../hbase/rsgroup/RSGroupBasedLoadBalancer.java |   431 +
 .../hadoop/hbase/rsgroup/RSGroupInfo.java       |   190 +
 .../hbase/rsgroup/RSGroupInfoManager.java       |   116 +
 .../hbase/rsgroup/RSGroupInfoManagerImpl.java   |   795 +
 .../hbase/rsgroup/RSGroupProtobufUtil.java      |    61 +
 .../hadoop/hbase/rsgroup/RSGroupSerDe.java      |    88 +
 .../hbase/rsgroup/RSGroupableBalancer.java      |    32 +
 .../balancer/TestRSGroupBasedLoadBalancer.java  |   573 +
 .../hadoop/hbase/rsgroup/TestRSGroups.java      |   300 +
 .../hadoop/hbase/rsgroup/TestRSGroupsBase.java  |   815 +
 .../hbase/rsgroup/TestRSGroupsOfflineMode.java  |   187 +
 .../rsgroup/VerifyingRSGroupAdminClient.java    |   155 +
 .../hbase/tmpl/master/MasterStatusTmpl.jamon    |     2 +
 .../apache/hadoop/hbase/LocalHBaseCluster.java  |     3 +
 .../BaseMasterAndRegionObserver.java            |    62 +
 .../hbase/coprocessor/BaseMasterObserver.java   |    63 +
 .../hbase/coprocessor/MasterObserver.java       |   113 +
 .../hadoop/hbase/master/AssignmentManager.java  |    16 +-
 .../org/apache/hadoop/hbase/master/HMaster.java |     5 +
 .../hadoop/hbase/master/LoadBalancer.java       |     3 +
 .../hbase/master/MasterCoprocessorHost.java     |   160 +
 .../hadoop/hbase/master/MasterServices.java     |     5 +
 .../hbase/security/access/AccessController.java |    37 +
 .../hbase/coprocessor/TestMasterObserver.java   |    61 +
 .../hbase/master/MockNoopMasterServices.java    |     5 +
 .../master/TestAssignmentManagerOnCluster.java  |   127 +-
 .../hadoop/hbase/master/TestCatalogJanitor.java |     3 +
 .../hbase/master/TestMasterStatusServlet.java   |    12 +-
 .../normalizer/TestSimpleRegionNormalizer.java  |     2 +-
 .../security/access/TestAccessController.java   |    75 +
 hbase-shell/pom.xml                             |    35 +
 hbase-shell/src/main/ruby/hbase.rb              |     1 +
 hbase-shell/src/main/ruby/hbase/hbase.rb        |     4 +
 .../src/main/ruby/hbase/rsgroup_admin.rb        |   164 +
 hbase-shell/src/main/ruby/shell.rb              |    22 +
 hbase-shell/src/main/ruby/shell/commands.rb     |     4 +
 .../src/main/ruby/shell/commands/add_rsgroup.rb |    39 +
 .../main/ruby/shell/commands/balance_rsgroup.rb |    37 +
 .../src/main/ruby/shell/commands/get_rsgroup.rb |    43 +
 .../ruby/shell/commands/get_server_rsgroup.rb   |    39 +
 .../ruby/shell/commands/get_table_rsgroup.rb    |    40 +
 .../main/ruby/shell/commands/list_rsgroups.rb   |    49 +
 .../ruby/shell/commands/move_servers_rsgroup.rb |    37 +
 .../commands/move_servers_tables_rsgroup.rb     |    37 +
 .../ruby/shell/commands/move_tables_rsgroup.rb  |    37 +
 .../main/ruby/shell/commands/remove_rsgroup.rb  |    37 +
 .../apache/hadoop/hbase/client/TestShell.java   |     2 +-
 .../hbase/client/rsgroup/TestShellRSGroups.java |   111 +
 .../src/test/ruby/shell/rsgroup_shell_test.rb   |    96 +
 hbase-shell/src/test/ruby/test_helper.rb        |     4 +
 pom.xml                                         |    23 +
 66 files changed, 22843 insertions(+), 21 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/c3200076/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerName.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerName.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerName.java
index c90e7e1..339f588 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerName.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerName.java
@@ -24,6 +24,7 @@ import com.google.protobuf.InvalidProtocolBufferException;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.classification.InterfaceStability;
 import org.apache.hadoop.hbase.exceptions.DeserializationException;
+import org.apache.hadoop.hbase.net.Address;
 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
 import org.apache.hadoop.hbase.util.Addressing;
@@ -106,7 +107,7 @@ public class ServerName implements Comparable<ServerName>, Serializable
{
     this.hostnameOnly = hostname;
     this.port = port;
     this.startcode = startcode;
-    this.servername = getServerName(this.hostnameOnly, port, startcode);
+    this.servername = getServerName(hostname, port, startcode);
   }
 
   /**
@@ -402,4 +403,26 @@ public class ServerName implements Comparable<ServerName>, Serializable
{
     int port = Addressing.parsePort(str);
     return valueOf(hostname, port, -1L);
   }
+
+  /**
+   * @return an Address constructed from the hostname and port carried by this ServerName
+   */
+  public Address getAddress() {
+    return Address.fromParts(getHostname(), getPort());
+  }
+
+  /**
+   * @param left
+   * @param right
+   * @return True if <code>other</code> has same hostname and port.
+   */
+  public static boolean isSameAddress(final ServerName left,
+                                      final ServerName right) {
+    // TODO: Make this left.getAddress().equals(right.getAddress())
+    if (left == null) return false;
+    if (right == null) return false;
+    return left.getHostname().compareToIgnoreCase(right.getHostname()) == 0 &&
+      left.getPort() == right.getPort();
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/c3200076/hbase-common/src/main/java/org/apache/hadoop/hbase/net/Address.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/net/Address.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/net/Address.java
new file mode 100644
index 0000000..15b4960
--- /dev/null
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/net/Address.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.net;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+import com.google.common.net.HostAndPort;
+
+/**
+ * An immutable type to hold a hostname and port combo, like an Endpoint
+ * or java.net.InetSocketAddress (but without danger of our calling
+ * resolve -- we do NOT want a resolve happening every time we want
+ * to hold a hostname and port combo). This class is also <<Comparable>>.
+ * <p>In implementation this class is a facade over Guava's {@link HostAndPort}.
+ * We cannot have Guava classes in our API hence this Type.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public class Address implements Comparable<Address> {
+  private HostAndPort hostAndPort;
+
+  private Address(HostAndPort hostAndPort) {
+    this.hostAndPort = hostAndPort;
+  }
+
+  public static Address fromParts(String hostname, int port) {
+    return new Address(HostAndPort.fromParts(hostname, port));
+  }
+
+  public static Address fromString(String hostnameAndPort) {
+    return new Address(HostAndPort.fromString(hostnameAndPort));
+  }
+
+  public String getHostname() {
+    return this.hostAndPort.getHostText();
+  }
+
+  public int getPort() {
+    return this.hostAndPort.getPort();
+  }
+
+  @Override
+  public String toString() {
+    return this.hostAndPort.toString();
+  }
+
+  @Override
+  // Don't use HostAndPort equals... It is wonky including
+  // ipv6 brackets
+  public boolean equals(Object other) {
+    if (this == other) {
+      return true;
+    }
+    if (other instanceof Address) {
+      Address that = (Address)other;
+      return this.getHostname().equals(that.getHostname()) &&
+          this.getPort() == that.getPort();
+    }
+    return false;
+  }
+
+  @Override
+  public int hashCode() {
+    return this.getHostname().hashCode() ^ getPort();
+  }
+
+  @Override
+  public int compareTo(Address that) {
+    int compare = this.getHostname().compareTo(that.getHostname());
+    if (compare != 0) return compare;
+    return this.getPort() - that.getPort();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/c3200076/hbase-it/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-it/pom.xml b/hbase-it/pom.xml
index 4c87bf1..46f251c 100644
--- a/hbase-it/pom.xml
+++ b/hbase-it/pom.xml
@@ -265,6 +265,47 @@
   </dependencies>
 
   <profiles>
+    <profile>
+      <id>rsgroup</id>
+      <activation>
+        <property>
+            <name>!skip-rsgroup</name>
+        </property>
+      </activation>
+      <dependencies>
+	<dependency>
+	  <groupId>org.apache.hbase</groupId>
+	  <artifactId>hbase-rsgroup</artifactId>
+	</dependency>
+	<dependency>
+	  <groupId>org.apache.hbase</groupId>
+	  <artifactId>hbase-rsgroup</artifactId>
+	  <type>test-jar</type>
+	  <scope>test</scope>
+	</dependency>
+      </dependencies>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.codehaus.mojo</groupId>
+            <artifactId>build-helper-maven-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>add-test-source</id>
+                <goals>
+                  <goal>add-test-source</goal>
+                </goals>
+                <configuration>
+                  <sources>
+                    <source>src/test/rsgroup</source>
+                  </sources>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
     <!-- Skip the tests in this module -->
     <profile>
       <id>skipIntegrationTests</id>

http://git-wip-us.apache.org/repos/asf/hbase/blob/c3200076/hbase-it/src/test/rsgroup/org/apache/hadoop/hbase/rsgroup/IntegrationTestRSGroup.java
----------------------------------------------------------------------
diff --git a/hbase-it/src/test/rsgroup/org/apache/hadoop/hbase/rsgroup/IntegrationTestRSGroup.java
b/hbase-it/src/test/rsgroup/org/apache/hadoop/hbase/rsgroup/IntegrationTestRSGroup.java
new file mode 100644
index 0000000..e5bb995
--- /dev/null
+++ b/hbase-it/src/test/rsgroup/org/apache/hadoop/hbase/rsgroup/IntegrationTestRSGroup.java
@@ -0,0 +1,99 @@
+/**
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.rsgroup;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.IntegrationTestingUtility;
+import org.apache.hadoop.hbase.Waiter;
+import org.apache.hadoop.hbase.testclassification.IntegrationTests;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Runs all of the units tests defined in TestGroupBase
+ * as an integration test.
+ * Requires TestRSGroupBase.NUM_SLAVE_BASE servers to run.
+ */
+@Category(IntegrationTests.class)
+public class IntegrationTestRSGroup extends TestRSGroupsBase {
+  //Integration specific
+  private final static Log LOG = LogFactory.getLog(IntegrationTestRSGroup.class);
+  private static boolean initialized = false;
+
+  @Before
+  public void beforeMethod() throws Exception {
+    if(!initialized) {
+      LOG.info("Setting up IntegrationTestGroup");
+      LOG.info("Initializing cluster with " + NUM_SLAVES_BASE + " servers");
+      TEST_UTIL = new IntegrationTestingUtility();
+      ((IntegrationTestingUtility)TEST_UTIL).initializeCluster(NUM_SLAVES_BASE);
+      //set shared configs
+      admin = TEST_UTIL.getHBaseAdmin();
+      cluster = TEST_UTIL.getHBaseClusterInterface();
+      rsGroupAdmin = new VerifyingRSGroupAdminClient(new RSGroupAdminClient(TEST_UTIL.getConnection()),
+          TEST_UTIL.getConfiguration());
+      LOG.info("Done initializing cluster");
+      initialized = true;
+      //cluster may not be clean
+      //cleanup when initializing
+      afterMethod();
+    }
+  }
+
+  @After
+  public void afterMethod() throws Exception {
+    LOG.info("Cleaning up previous test run");
+    //cleanup previous artifacts
+    deleteTableIfNecessary();
+    deleteNamespaceIfNecessary();
+    deleteGroups();
+    admin.setBalancerRunning(true, true);
+
+    LOG.info("Restoring the cluster");
+    ((IntegrationTestingUtility)TEST_UTIL).restoreCluster();
+    LOG.info("Done restoring the cluster");
+
+    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
+      @Override
+      public boolean evaluate() throws Exception {
+        LOG.info("Waiting for cleanup to finish "+ rsGroupAdmin.listRSGroups());
+        //Might be greater since moving servers back to default
+        //is after starting a server
+        return rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size()
+            >= NUM_SLAVES_BASE;
+      }
+    });
+
+    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
+      @Override
+      public boolean evaluate() throws Exception {
+        LOG.info("Waiting for regionservers to be registered "+ rsGroupAdmin.listRSGroups());
+        //Might be greater since moving servers back to default
+        //is after starting a server
+        return rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size()
+            == getNumServers();
+      }
+    });
+
+    LOG.info("Done cleaning up previous test run");
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/c3200076/hbase-protocol/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-protocol/pom.xml b/hbase-protocol/pom.xml
index 496b3cf..a389d78 100644
--- a/hbase-protocol/pom.xml
+++ b/hbase-protocol/pom.xml
@@ -195,6 +195,8 @@
                           <include>RegionNormalizer.proto</include>
                           <include>RegionServerStatus.proto</include>
                           <include>RowProcessor.proto</include>
+                          <include>RSGroup.proto</include>
+                          <include>RSGroupAdmin.proto</include>
                           <include>SecureBulkLoad.proto</include>
                           <include>Snapshot.proto</include>
                           <include>Table.proto</include>

http://git-wip-us.apache.org/repos/asf/hbase/blob/c3200076/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClientProtos.java
----------------------------------------------------------------------
diff --git a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClientProtos.java
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClientProtos.java
index 06ebc65..35fddc2 100644
--- a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClientProtos.java
+++ b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClientProtos.java
@@ -1986,7 +1986,7 @@ public final class ClientProtos {
      * <code>optional bool load_column_families_on_demand = 14;</code>
      *
      * <pre>
-     * DO NOT add defaults to load_column_families_on_demand.
+     * DO NOT add defaults to load_column_families_on_demand. 
      * </pre>
      */
     boolean hasLoadColumnFamiliesOnDemand();
@@ -1994,7 +1994,7 @@ public final class ClientProtos {
      * <code>optional bool load_column_families_on_demand = 14;</code>
      *
      * <pre>
-     * DO NOT add defaults to load_column_families_on_demand.
+     * DO NOT add defaults to load_column_families_on_demand. 
      * </pre>
      */
     boolean getLoadColumnFamiliesOnDemand();
@@ -2515,7 +2515,7 @@ public final class ClientProtos {
      * <code>optional bool load_column_families_on_demand = 14;</code>
      *
      * <pre>
-     * DO NOT add defaults to load_column_families_on_demand.
+     * DO NOT add defaults to load_column_families_on_demand. 
      * </pre>
      */
     public boolean hasLoadColumnFamiliesOnDemand() {
@@ -2525,7 +2525,7 @@ public final class ClientProtos {
      * <code>optional bool load_column_families_on_demand = 14;</code>
      *
      * <pre>
-     * DO NOT add defaults to load_column_families_on_demand.
+     * DO NOT add defaults to load_column_families_on_demand. 
      * </pre>
      */
     public boolean getLoadColumnFamiliesOnDemand() {
@@ -4577,7 +4577,7 @@ public final class ClientProtos {
        * <code>optional bool load_column_families_on_demand = 14;</code>
        *
        * <pre>
-       * DO NOT add defaults to load_column_families_on_demand.
+       * DO NOT add defaults to load_column_families_on_demand. 
        * </pre>
        */
       public boolean hasLoadColumnFamiliesOnDemand() {
@@ -4587,7 +4587,7 @@ public final class ClientProtos {
        * <code>optional bool load_column_families_on_demand = 14;</code>
        *
        * <pre>
-       * DO NOT add defaults to load_column_families_on_demand.
+       * DO NOT add defaults to load_column_families_on_demand. 
        * </pre>
        */
       public boolean getLoadColumnFamiliesOnDemand() {
@@ -4597,7 +4597,7 @@ public final class ClientProtos {
        * <code>optional bool load_column_families_on_demand = 14;</code>
        *
        * <pre>
-       * DO NOT add defaults to load_column_families_on_demand.
+       * DO NOT add defaults to load_column_families_on_demand. 
        * </pre>
        */
       public Builder setLoadColumnFamiliesOnDemand(boolean value) {
@@ -4610,7 +4610,7 @@ public final class ClientProtos {
        * <code>optional bool load_column_families_on_demand = 14;</code>
        *
        * <pre>
-       * DO NOT add defaults to load_column_families_on_demand.
+       * DO NOT add defaults to load_column_families_on_demand. 
        * </pre>
        */
       public Builder clearLoadColumnFamiliesOnDemand() {


Mime
View raw message