hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From te...@apache.org
Subject hbase git commit: HBASE-17460 enable_table_replication can not perform cyclic replication of a table (Janos Gub)
Date Wed, 08 Mar 2017 09:27:42 GMT
Repository: hbase
Updated Branches:
  refs/heads/branch-1 dcaa9bd71 -> f34709e1b


HBASE-17460 enable_table_replication can not perform cyclic replication of a table (Janos
Gub)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f34709e1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f34709e1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f34709e1

Branch: refs/heads/branch-1
Commit: f34709e1b77c30d938d2c3a73375f2106378a1a6
Parents: dcaa9bd
Author: tedyu <yuzhihong@gmail.com>
Authored: Wed Mar 8 01:27:35 2017 -0800
Committer: tedyu <yuzhihong@gmail.com>
Committed: Wed Mar 8 01:27:35 2017 -0800

----------------------------------------------------------------------
 .../apache/hadoop/hbase/HColumnDescriptor.java  |  2 +-
 .../apache/hadoop/hbase/HTableDescriptor.java   |  1 +
 .../client/replication/ReplicationAdmin.java    | 92 ++++++++++++++++++--
 .../hadoop/hbase/backup/HFileArchiver.java      |  2 +-
 .../TestReplicationAdminWithClusters.java       |  1 +
 5 files changed, 88 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/f34709e1/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
index 3b2e99c..94899e3 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
@@ -1386,7 +1386,7 @@ public class HColumnDescriptor implements WritableComparable<HColumnDescriptor>
   public int compareTo(HColumnDescriptor o) {
     int result = Bytes.compareTo(this.name, o.getName());
     if (result == 0) {
-      // punt on comparison for ordering, just calculate difference
+      // punt on comparison for ordering, just calculate difference.
       result = this.values.hashCode() - o.values.hashCode();
       if (result < 0)
         result = -1;

http://git-wip-us.apache.org/repos/asf/hbase/blob/f34709e1/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
index 4f3ac9c..ba0c126 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
@@ -1063,6 +1063,7 @@ public class HTableDescriptor implements WritableComparable<HTableDescriptor>
{
     return compareTo((HTableDescriptor)obj) == 0;
   }
 
+
   /**
    * @see java.lang.Object#hashCode()
    */

http://git-wip-us.apache.org/repos/asf/hbase/blob/f34709e1/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
index d462f38..f9cf6b3 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
@@ -18,19 +18,18 @@
  */
 package org.apache.hadoop.hbase.client.replication;
 
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.Lists;
-
 import java.io.Closeable;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
+import java.util.TreeMap;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
@@ -54,6 +53,9 @@ import org.apache.hadoop.hbase.replication.ReplicationSerDeHelper;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Lists;
+
 /**
  * <p>
  * This class provides the administrative interface to HBase cluster
@@ -560,7 +562,9 @@ public class ReplicationAdmin implements Closeable {
    * Connect to peer and check the table descriptor on peer:
    * <ol>
    * <li>Create the same table on peer when not exist.</li>
-   * <li>Throw exception if the table exists on peer cluster but descriptors are not
same.</li>
+   * <li>Throw an exception if the table already has replication enabled on any of
the column
+   * families.</li>
+   * <li>Throw an exception if the table exists on peer cluster but descriptors are
not same.</li>
    * </ol>
    * @param tableName name of the table to sync to the peer
    * @param splits table split keys
@@ -584,20 +588,21 @@ public class ReplicationAdmin implements Closeable {
       }
 
       Configuration peerConf = repPeer.getConfiguration();
-      HTableDescriptor htd = null;
+      HTableDescriptor localHtd = null;
       try (Connection conn = ConnectionFactory.createConnection(peerConf);
           Admin admin = this.connection.getAdmin();
           Admin repHBaseAdmin = conn.getAdmin()) {
-        htd = admin.getTableDescriptor(tableName);
+        localHtd = admin.getTableDescriptor(tableName);
         HTableDescriptor peerHtd = null;
         if (!repHBaseAdmin.tableExists(tableName)) {
-          repHBaseAdmin.createTable(htd, splits);
+          repHBaseAdmin.createTable(localHtd, splits);
         } else {
           peerHtd = repHBaseAdmin.getTableDescriptor(tableName);
           if (peerHtd == null) {
             throw new IllegalArgumentException("Failed to get table descriptor for table
"
                 + tableName.getNameAsString() + " from peer cluster " + repPeer.getId());
-          } else if (!peerHtd.equals(htd)) {
+          }
+          if (!compareForReplication(peerHtd, localHtd)) {
             throw new IllegalArgumentException("Table " + tableName.getNameAsString()
                 + " exists in peer cluster " + repPeer.getId()
                 + ", but the table descriptors are not same when compared with source cluster."
@@ -709,4 +714,75 @@ public class ReplicationAdmin implements Closeable {
       }
     }
   }
+
+  /**
+   * Copies the REPLICATION_SCOPE of table descriptor passed as an argument. Before copy,
the method
+   * ensures that the name of table and column-families should match.
+   * @param peerHtd descriptor on peer cluster
+   * @param localHtd - The HTableDescriptor of table from source cluster.
+   * @return true If the name of table and column families match and REPLICATION_SCOPE copied
+   *         successfully. false If there is any mismatch in the names.
+   */
+  private boolean copyReplicationScope(final HTableDescriptor peerHtd,
+      final HTableDescriptor localHtd) {
+    // Copy the REPLICATION_SCOPE only when table names and the names of
+    // Column-Families are same.
+    int result = peerHtd.getTableName().compareTo(localHtd.getTableName());
+
+    if (result == 0) {
+      Iterator<HColumnDescriptor> remoteHCDIter = peerHtd.getFamilies().iterator();
+      Iterator<HColumnDescriptor> localHCDIter = localHtd.getFamilies().iterator();
+
+      while (remoteHCDIter.hasNext() && localHCDIter.hasNext()) {
+        HColumnDescriptor remoteHCD = remoteHCDIter.next();
+        HColumnDescriptor localHCD = localHCDIter.next();
+
+        String remoteHCDName = remoteHCD.getNameAsString();
+        String localHCDName = localHCD.getNameAsString();
+
+        if (remoteHCDName.equals(localHCDName)) {
+          remoteHCD.setScope(localHCD.getScope());
+        } else {
+          result = -1;
+          break;
+        }
+      }
+      if (remoteHCDIter.hasNext() || localHCDIter.hasNext()) {
+        return false;
+      }
+    }
+
+    return result == 0;
+  }
+
+  /**
+   * Compare the contents of the descriptor with another one passed as a parameter for replication
+   * purpose. The REPLICATION_SCOPE field is ignored during comparison.
+   * @param peerHtd descriptor on peer cluster
+   * @param localHtd descriptor on source cluster which needs to be replicated.
+   * @return true if the contents of the two descriptors match (ignoring just REPLICATION_SCOPE).
+   * @see java.lang.Object#equals(java.lang.Object)
+   */
+  private boolean compareForReplication(HTableDescriptor peerHtd, HTableDescriptor localHtd)
{
+    if (peerHtd == localHtd) {
+      return true;
+    }
+    if (peerHtd == null) {
+      return false;
+    }
+    boolean result = false;
+
+    // Create a copy of peer HTD as we need to change its replication
+    // scope to match with the local HTD.
+    HTableDescriptor peerHtdCopy = new HTableDescriptor(peerHtd);
+
+    result = copyReplicationScope(peerHtdCopy, localHtd);
+
+    // If copy was successful, compare the two tables now.
+    if (result) {
+      result = (peerHtdCopy.compareTo(localHtd) == 0);
+    }
+
+    return result;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/f34709e1/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
index ff45f11..1231173 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
@@ -51,7 +51,7 @@ import com.google.common.collect.Lists;
 /**
  * Utility class to handle the removal of HFiles (or the respective {@link StoreFile StoreFiles})
  * for a HRegion from the {@link FileSystem}. The hfiles will be archived or deleted, depending
on
- * the state of the system.
+ * the state of the system. 
  */
 @InterfaceAudience.Private
 public class HFileArchiver {

http://git-wip-us.apache.org/repos/asf/hbase/blob/f34709e1/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminWithClusters.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminWithClusters.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminWithClusters.java
index b75c1cf..26ed7a7 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminWithClusters.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminWithClusters.java
@@ -71,6 +71,7 @@ public class TestReplicationAdminWithClusters extends TestReplicationBase
{
 
   @Test(timeout = 300000)
   public void testEnableReplicationWhenSlaveClusterDoesntHaveTable() throws Exception {
+    admin.disableTableRep(tableName);
     admin2.disableTable(tableName);
     admin2.deleteTable(tableName);
     assertFalse(admin2.tableExists(tableName));


Mime
View raw message