hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zhang...@apache.org
Subject [hbase] 06/07: HBASE-22820 Do not need to persist default rs group now (#482)
Date Sun, 01 Sep 2019 14:14:28 GMT
This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch HBASE-22514
in repository https://gitbox.apache.org/repos/asf/hbase.git

commit fc209a42938873e1aa8d7dba346f1a5db4fcd8c2
Author: linkaline <linkaline11@gmail.com>
AuthorDate: Fri Aug 16 16:52:41 2019 +0800

    HBASE-22820 Do not need to persist default rs group now (#482)
    
    Signed-off-by: Duo Zhang <zhangduo@apache.org>
---
 .../hbase/rsgroup/RSGroupInfoManagerImpl.java      | 40 +++++++++++++---------
 .../hbase/rsgroup/VerifyingRSGroupAdminClient.java | 25 ++++++++++++++
 2 files changed, 49 insertions(+), 16 deletions(-)

diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java
index 37f3ce6..eaf23f3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java
@@ -396,11 +396,13 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager {
     }
 
     // populate puts
-    for (RSGroupInfo RSGroupInfo : groupMap.values()) {
-      RSGroupProtos.RSGroupInfo proto = ProtobufUtil.toProtoGroupInfo(RSGroupInfo);
-      Put p = new Put(Bytes.toBytes(RSGroupInfo.getName()));
-      p.addColumn(META_FAMILY_BYTES, META_QUALIFIER_BYTES, proto.toByteArray());
-      mutations.add(p);
+    for (RSGroupInfo gi : groupMap.values()) {
+      if (!gi.getName().equals(RSGroupInfo.DEFAULT_GROUP)) {
+        RSGroupProtos.RSGroupInfo proto = ProtobufUtil.toProtoGroupInfo(gi);
+        Put p = new Put(Bytes.toBytes(gi.getName()));
+        p.addColumn(META_FAMILY_BYTES, META_QUALIFIER_BYTES, proto.toByteArray());
+        mutations.add(p);
+      }
     }
 
     if (mutations.size() > 0) {
@@ -449,7 +451,12 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager {
 
     // Make changes visible after having been persisted to the source of truth
     resetRSGroupMap(newGroupMap);
+    saveRSGroupMapToZK(newGroupMap);
 
+    updateCacheOfRSGroups(newGroupMap.keySet());
+  }
+
+  private void saveRSGroupMapToZK(Map<String, RSGroupInfo> newGroupMap) throws IOException
{
     try {
       String groupBasePath =
           ZNodePaths.joinZNode(watcher.getZNodePaths().baseZNode, RS_GROUP_ZNODE);
@@ -463,14 +470,16 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager {
         }
       }
 
-      for (RSGroupInfo RSGroupInfo : newGroupMap.values()) {
-        String znode = ZNodePaths.joinZNode(groupBasePath, RSGroupInfo.getName());
-        RSGroupProtos.RSGroupInfo proto = ProtobufUtil.toProtoGroupInfo(RSGroupInfo);
-        LOG.debug("Updating znode: " + znode);
-        ZKUtil.createAndFailSilent(watcher, znode);
-        zkOps.add(ZKUtil.ZKUtilOp.deleteNodeFailSilent(znode));
-        zkOps.add(ZKUtil.ZKUtilOp.createAndFailSilent(znode,
-          ProtobufUtil.prependPBMagic(proto.toByteArray())));
+      for (RSGroupInfo gi : newGroupMap.values()) {
+        if (!gi.getName().equals(RSGroupInfo.DEFAULT_GROUP)) {
+          String znode = ZNodePaths.joinZNode(groupBasePath, gi.getName());
+          RSGroupProtos.RSGroupInfo proto = ProtobufUtil.toProtoGroupInfo(gi);
+          LOG.debug("Updating znode: " + znode);
+          ZKUtil.createAndFailSilent(watcher, znode);
+          zkOps.add(ZKUtil.ZKUtilOp.deleteNodeFailSilent(znode));
+          zkOps.add(ZKUtil.ZKUtilOp.createAndFailSilent(znode,
+              ProtobufUtil.prependPBMagic(proto.toByteArray())));
+        }
       }
       LOG.debug("Writing ZK GroupInfo count: " + zkOps.size());
 
@@ -480,7 +489,6 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager {
       masterServices.abort("Failed to write to rsGroupZNode", e);
       throw new IOException("Failed to write to rsGroupZNode", e);
     }
-    updateCacheOfRSGroups(newGroupMap.keySet());
   }
 
   /**
@@ -540,12 +548,12 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager {
 
   // Called by ServerEventsListenerThread. Synchronize on this because redoing
   // the rsGroupMap then writing it out.
-  private synchronized void updateDefaultServers(SortedSet<Address> servers) throws
IOException {
+  private synchronized void updateDefaultServers(SortedSet<Address> servers) {
     RSGroupInfo info = rsGroupMap.get(RSGroupInfo.DEFAULT_GROUP);
     RSGroupInfo newInfo = new RSGroupInfo(info.getName(), servers);
     HashMap<String, RSGroupInfo> newGroupMap = Maps.newHashMap(rsGroupMap);
     newGroupMap.put(newInfo.getName(), newInfo);
-    flushConfig(newGroupMap);
+    resetRSGroupMap(newGroupMap);
   }
 
   /**
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdminClient.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdminClient.java
index a8cd277..28131a9 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdminClient.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdminClient.java
@@ -26,8 +26,11 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.SortedSet;
+import java.util.regex.Pattern;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.Admin;
 import org.apache.hadoop.hbase.client.Connection;
@@ -133,6 +136,13 @@ public class VerifyingRSGroupAdminClient extends RSGroupAdminClient {
       tds.addAll(admin.listTableDescriptors());
       tds.addAll(admin.listTableDescriptorsByNamespace(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME));
     }
+    SortedSet<Address> lives = Sets.newTreeSet();
+    for (ServerName sn : conn.getAdmin().getClusterMetrics().getLiveServerMetrics().keySet())
{
+      lives.add(sn.getAddress());
+    }
+    for (ServerName sn : conn.getAdmin().listDecommissionedRegionServers()) {
+      lives.remove(sn.getAddress());
+    }
     try (Table table = conn.getTable(RSGroupInfoManagerImpl.RSGROUP_TABLE_NAME);
         ResultScanner scanner = table.getScanner(new Scan())) {
       for (;;) {
@@ -144,8 +154,22 @@ public class VerifyingRSGroupAdminClient extends RSGroupAdminClient {
           RSGroupInfoManagerImpl.META_FAMILY_BYTES, RSGroupInfoManagerImpl.META_QUALIFIER_BYTES));
         RSGroupInfo rsGroupInfo = ProtobufUtil.toGroupInfo(proto);
         groupMap.put(proto.getName(), RSGroupUtil.fillTables(rsGroupInfo, tds));
+        for(Address address : rsGroupInfo.getServers()){
+          lives.remove(address);
+        }
       }
     }
+    SortedSet<TableName> tables = Sets.newTreeSet();
+    for (TableDescriptor td : conn.getAdmin().listTableDescriptors(Pattern.compile(".*"),
+        true)){
+      String groupName = td.getRegionServerGroup().orElse(RSGroupInfo.DEFAULT_GROUP);
+      if (groupName.equals(RSGroupInfo.DEFAULT_GROUP)) {
+        tables.add(td.getTableName());
+      }
+    }
+
+    groupMap.put(RSGroupInfo.DEFAULT_GROUP,
+        new RSGroupInfo(RSGroupInfo.DEFAULT_GROUP, lives, tables));
     assertEquals(Sets.newHashSet(groupMap.values()), Sets.newHashSet(wrapped.listRSGroups()));
     try {
       String groupBasePath = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "rsgroup");
@@ -160,6 +184,7 @@ public class VerifyingRSGroupAdminClient extends RSGroupAdminClient {
           zList.add(RSGroupUtil.fillTables(rsGroupInfo, tds));
         }
       }
+      groupMap.remove(RSGroupInfo.DEFAULT_GROUP);
       assertEquals(zList.size(), groupMap.size());
       for (RSGroupInfo rsGroupInfo : zList) {
         assertTrue(groupMap.get(rsGroupInfo.getName()).equals(rsGroupInfo));


Mime
View raw message