hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From te...@apache.org
Subject hbase git commit: HBASE-17460 enable_table_replication can not perform cyclic replication of a table (NITIN VERMA)
Date Sat, 18 Feb 2017 01:56:58 GMT
Repository: hbase
Updated Branches:
  refs/heads/master b392de3e3 -> 0e05537de


HBASE-17460 enable_table_replication can not perform cyclic replication of a table (NITIN
VERMA)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0e05537d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0e05537d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0e05537d

Branch: refs/heads/master
Commit: 0e05537dec32d31e1d025ddf60276df01ea0e28b
Parents: b392de3
Author: tedyu <yuzhihong@gmail.com>
Authored: Fri Feb 17 17:56:53 2017 -0800
Committer: tedyu <yuzhihong@gmail.com>
Committed: Fri Feb 17 17:56:53 2017 -0800

----------------------------------------------------------------------
 .../apache/hadoop/hbase/HColumnDescriptor.java  |   2 +-
 .../apache/hadoop/hbase/HTableDescriptor.java   | 102 ++++++++++++++++++-
 .../apache/hadoop/hbase/client/HBaseAdmin.java  |  31 ++++--
 .../client/replication/ReplicationAdmin.java    |   8 +-
 .../TestReplicationAdminWithClusters.java       |   1 +
 5 files changed, 129 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/0e05537d/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
index 1597a06..6d1ae3f 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
@@ -1076,7 +1076,7 @@ public class HColumnDescriptor implements Comparable<HColumnDescriptor>
{
   public int compareTo(HColumnDescriptor o) {
     int result = Bytes.compareTo(this.name, o.getName());
     if (result == 0) {
-      // punt on comparison for ordering, just calculate difference
+      // punt on comparison for ordering, just calculate difference.
       result = this.values.hashCode() - o.values.hashCode();
       if (result < 0)
         result = -1;

http://git-wip-us.apache.org/repos/asf/hbase/blob/0e05537d/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
index 60b85fe..a81c82a 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
@@ -40,9 +40,9 @@ import org.apache.hadoop.hbase.classification.InterfaceStability;
 import org.apache.hadoop.hbase.client.Durability;
 import org.apache.hadoop.hbase.client.RegionReplicaUtil;
 import org.apache.hadoop.hbase.exceptions.DeserializationException;
+import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.TableSchema;
-import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.Bytes;
 
 /**
@@ -1042,6 +1042,106 @@ public class HTableDescriptor implements Comparable<HTableDescriptor>
{
   }
 
   /**
+   * Detects whether replication has been already enabled on any of the column families of
this
+   * table descriptor.
+   * @return true if any of the column families has replication enabled.
+   */
+  public boolean isReplicationEnabled() {
+    // Go through each Column-Family descriptor and check if the
+    // Replication has been enabled already.
+    // Return 'true' if replication has been enabled on any CF,
+    // otherwise return 'false'.
+    //
+    boolean result = false;
+    Iterator<HColumnDescriptor> it = this.families.values().iterator();
+
+    while (it.hasNext()) {
+      HColumnDescriptor tempHcd = it.next();
+      if (tempHcd.getScope() != HConstants.REPLICATION_SCOPE_LOCAL) {
+        result = true;
+        break;
+      }
+    }
+
+    return result;
+  }
+
+  /**
+   * Compare the contents of the descriptor with another one passed as a parameter for replication
+   * purpose. The REPLICATION_SCOPE field is ignored during comparison.
+   * @param obj descriptor on source cluster which needs to be replicated.
+   * @return true if the contents of the two descriptors match (ignoring just REPLICATION_SCOPE).
+   * @see java.lang.Object#equals(java.lang.Object)
+   */
+  public boolean compareForReplication(Object obj) {
+    if (this == obj) {
+      return true;
+    }
+    if (obj == null) {
+      return false;
+    }
+    if (!(obj instanceof HTableDescriptor)) {
+      return false;
+    }
+
+    boolean result = false;
+
+    // Create a copy of peer HTD as we need to change its replication
+    // scope to match with the local HTD.
+    HTableDescriptor peerHtdCopy = new HTableDescriptor(this);
+
+    // Copy the replication scope of local Htd to remote Htd.
+    HTableDescriptor localHtd = (HTableDescriptor) obj;
+
+    result = (peerHtdCopy.copyReplicationScope(localHtd) == 0);
+
+    // If copy was successful, compare the two tables now.
+    if (result == true) {
+      result = (peerHtdCopy.compareTo(localHtd) == 0);
+    }
+
+    return result;
+  }
+
+  /**
+   * Copies the REPLICATION_SCOPE of table descriptor passed as an argument. Before copy,
the method
+   * ensures that the name of table and column-families should match.
+   * @param localHtd - The HTableDescriptor of table from source cluster.
+   * @return 0 If the name of table and column families match and REPLICATION_SCOPE copied
+   *         successfully. 1 If there is any mismatch in the names.
+   */
+  public int copyReplicationScope(final HTableDescriptor localHtd)
+  {
+    // Copy the REPLICATION_SCOPE only when table names and the names of
+    // Column-Families are same.
+    int result = this.name.compareTo(localHtd.name);
+
+    if (result == 0) {
+      Iterator<HColumnDescriptor> remoteHCDIter = families.values().iterator();
+      Iterator<HColumnDescriptor> localHCDIter = localHtd.families.values().iterator();
+          
+      while (remoteHCDIter.hasNext() && localHCDIter.hasNext()) {
+        HColumnDescriptor remoteHCD = remoteHCDIter.next();
+        HColumnDescriptor localHCD = localHCDIter.next();
+        
+        String remoteHCDName = remoteHCD.getNameAsString();
+        String localHCDName = localHCD.getNameAsString();
+
+        if (remoteHCDName.equals(localHCDName))
+        {
+          remoteHCD.setScope(localHCD.getScope());
+        }
+        else {
+          result = -1;
+          break;
+        }
+      }
+    }
+
+    return result;
+  }
+
+  /**
    * @see java.lang.Object#hashCode()
    */
   @Override

http://git-wip-us.apache.org/repos/asf/hbase/blob/0e05537d/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index 65070b9..6fb448b 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -29,7 +29,6 @@ import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.Map.Entry;
 import java.util.Set;
 import java.util.TreeMap;
@@ -4087,7 +4086,9 @@ public class HBaseAdmin implements Admin {
    * Connect to peer and check the table descriptor on peer:
    * <ol>
    * <li>Create the same table on peer when not exist.</li>
-   * <li>Throw exception if the table exists on peer cluster but descriptors are not
same.</li>
+   * <li>Throw an exception if the table already has replication enabled on any of
the column
+   * families.</li>
+   * <li>Throw an exception if the table exists on peer cluster but descriptors are
not same.</li>
    * </ol>
    * @param tableName name of the table to sync to the peer
    * @param splits table split keys
@@ -4105,20 +4106,32 @@ public class HBaseAdmin implements Admin {
         Configuration peerConf = getPeerClusterConfiguration(peerDesc);
         try (Connection conn = ConnectionFactory.createConnection(peerConf);
             Admin repHBaseAdmin = conn.getAdmin()) {
-          HTableDescriptor htd = getTableDescriptor(tableName);
+          HTableDescriptor localHtd = getTableDescriptor(tableName);
           HTableDescriptor peerHtd = null;
           if (!repHBaseAdmin.tableExists(tableName)) {
-            repHBaseAdmin.createTable(htd, splits);
+            repHBaseAdmin.createTable(localHtd, splits);
           } else {
             peerHtd = repHBaseAdmin.getTableDescriptor(tableName);
             if (peerHtd == null) {
               throw new IllegalArgumentException("Failed to get table descriptor for table
"
                   + tableName.getNameAsString() + " from peer cluster " + peerDesc.getPeerId());
-            } else if (!peerHtd.equals(htd)) {
-              throw new IllegalArgumentException("Table " + tableName.getNameAsString()
-                  + " exists in peer cluster " + peerDesc.getPeerId()
-                  + ", but the table descriptors are not same when compared with source cluster."
-                  + " Thus can not enable the table's replication switch.");
+            } else {
+              // To support cyclic replication (HBASE-17460), we need to match the
+              // REPLICATION_SCOPE of table on both the clusters. We should do this
+              // only when the replication is not already enabled on local HTD (local
+              // table on this cluster).
+              //
+              if (localHtd.isReplicationEnabled()) {
+                throw new IllegalArgumentException("Table " + tableName.getNameAsString()
+                    + " has replication already enabled for at least one Column Family.");
+              } else {
+                if (!peerHtd.compareForReplication(localHtd)) {
+                  throw new IllegalArgumentException("Table " + tableName.getNameAsString()
+                      + " exists in peer cluster " + peerDesc.getPeerId()
+                      + ", but the table descriptors are not same when compared with source
cluster."
+                      + " Thus can not enable the table's replication switch.");
+                }
+              }
             }
           }
         }

http://git-wip-us.apache.org/repos/asf/hbase/blob/0e05537d/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
index 706f81e..4e74d87 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
@@ -18,9 +18,6 @@
  */
 package org.apache.hadoop.hbase.client.replication;
 
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.Lists;
-
 import java.io.Closeable;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -29,9 +26,9 @@ import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.TreeMap;
 import java.util.Map.Entry;
 import java.util.Set;
+import java.util.TreeMap;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -62,6 +59,9 @@ import org.apache.hadoop.hbase.replication.ReplicationQueuesClientArguments;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Lists;
+
 /**
  * <p>
  * This class provides the administrative interface to HBase cluster

http://git-wip-us.apache.org/repos/asf/hbase/blob/0e05537d/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminWithClusters.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminWithClusters.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminWithClusters.java
index 96fca47..24889ad 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminWithClusters.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminWithClusters.java
@@ -77,6 +77,7 @@ public class TestReplicationAdminWithClusters extends TestReplicationBase
{
 
   @Test(timeout = 300000)
   public void testEnableReplicationWhenSlaveClusterDoesntHaveTable() throws Exception {
+    admin1.disableTableReplication(tableName);
     admin2.disableTable(tableName);
     admin2.deleteTable(tableName);
     assertFalse(admin2.tableExists(tableName));


Mime
View raw message