Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id DA501200C31 for ; Wed, 8 Mar 2017 10:27:43 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id D9009160B83; Wed, 8 Mar 2017 09:27:43 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 09170160B75 for ; Wed, 8 Mar 2017 10:27:42 +0100 (CET) Received: (qmail 85038 invoked by uid 500); 8 Mar 2017 09:27:42 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 85028 invoked by uid 99); 8 Mar 2017 09:27:42 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 08 Mar 2017 09:27:42 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 1F2BDDFBD3; Wed, 8 Mar 2017 09:27:42 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: tedyu@apache.org To: commits@hbase.apache.org Message-Id: <426e2a2f29264868a413f7acceb4fa96@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hbase git commit: HBASE-17460 enable_table_replication can not perform cyclic replication of a table (Janos Gub) Date: Wed, 8 Mar 2017 09:27:42 +0000 (UTC) archived-at: Wed, 08 Mar 2017 09:27:44 -0000 Repository: hbase Updated Branches: refs/heads/branch-1 dcaa9bd71 -> f34709e1b HBASE-17460 enable_table_replication can not perform cyclic replication of a table (Janos Gub) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f34709e1 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f34709e1 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f34709e1 Branch: refs/heads/branch-1 Commit: f34709e1b77c30d938d2c3a73375f2106378a1a6 Parents: dcaa9bd Author: tedyu Authored: Wed Mar 8 01:27:35 2017 -0800 Committer: tedyu Committed: Wed Mar 8 01:27:35 2017 -0800 ---------------------------------------------------------------------- .../apache/hadoop/hbase/HColumnDescriptor.java | 2 +- .../apache/hadoop/hbase/HTableDescriptor.java | 1 + .../client/replication/ReplicationAdmin.java | 92 ++++++++++++++++++-- .../hadoop/hbase/backup/HFileArchiver.java | 2 +- .../TestReplicationAdminWithClusters.java | 1 + 5 files changed, 88 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/f34709e1/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java index 3b2e99c..94899e3 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java @@ -1386,7 +1386,7 @@ public class HColumnDescriptor implements WritableComparable public int compareTo(HColumnDescriptor o) { int result = Bytes.compareTo(this.name, o.getName()); if (result == 0) { - // punt on comparison for ordering, just calculate difference + // punt on comparison for ordering, just calculate difference. result = this.values.hashCode() - o.values.hashCode(); if (result < 0) result = -1; http://git-wip-us.apache.org/repos/asf/hbase/blob/f34709e1/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java index 4f3ac9c..ba0c126 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java @@ -1063,6 +1063,7 @@ public class HTableDescriptor implements WritableComparable { return compareTo((HTableDescriptor)obj) == 0; } + /** * @see java.lang.Object#hashCode() */ http://git-wip-us.apache.org/repos/asf/hbase/blob/f34709e1/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java index d462f38..f9cf6b3 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java @@ -18,19 +18,18 @@ */ package org.apache.hadoop.hbase.client.replication; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Lists; - import java.io.Closeable; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import java.util.TreeMap; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; @@ -54,6 +53,9 @@ import org.apache.hadoop.hbase.replication.ReplicationSerDeHelper; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Lists; + /** *

* This class provides the administrative interface to HBase cluster @@ -560,7 +562,9 @@ public class ReplicationAdmin implements Closeable { * Connect to peer and check the table descriptor on peer: *

    *
  1. Create the same table on peer when not exist.
    2. - *
  2. Throw exception if the table exists on peer cluster but descriptors are not same.
    3. + *
  3. Throw an exception if the table already has replication enabled on any of the column + * families.
    4. + *
  4. Throw an exception if the table exists on peer cluster but descriptors are not same.
    5. *
* @param tableName name of the table to sync to the peer * @param splits table split keys @@ -584,20 +588,21 @@ public class ReplicationAdmin implements Closeable { } Configuration peerConf = repPeer.getConfiguration(); - HTableDescriptor htd = null; + HTableDescriptor localHtd = null; try (Connection conn = ConnectionFactory.createConnection(peerConf); Admin admin = this.connection.getAdmin(); Admin repHBaseAdmin = conn.getAdmin()) { - htd = admin.getTableDescriptor(tableName); + localHtd = admin.getTableDescriptor(tableName); HTableDescriptor peerHtd = null; if (!repHBaseAdmin.tableExists(tableName)) { - repHBaseAdmin.createTable(htd, splits); + repHBaseAdmin.createTable(localHtd, splits); } else { peerHtd = repHBaseAdmin.getTableDescriptor(tableName); if (peerHtd == null) { throw new IllegalArgumentException("Failed to get table descriptor for table " + tableName.getNameAsString() + " from peer cluster " + repPeer.getId()); - } else if (!peerHtd.equals(htd)) { + } + if (!compareForReplication(peerHtd, localHtd)) { throw new IllegalArgumentException("Table " + tableName.getNameAsString() + " exists in peer cluster " + repPeer.getId() + ", but the table descriptors are not same when compared with source cluster." @@ -709,4 +714,75 @@ public class ReplicationAdmin implements Closeable { } } } + + /** + * Copies the REPLICATION_SCOPE of table descriptor passed as an argument. Before copy, the method + * ensures that the name of table and column-families should match. + * @param peerHtd descriptor on peer cluster + * @param localHtd - The HTableDescriptor of table from source cluster. + * @return true If the name of table and column families match and REPLICATION_SCOPE copied + * successfully. false If there is any mismatch in the names. + */ + private boolean copyReplicationScope(final HTableDescriptor peerHtd, + final HTableDescriptor localHtd) { + // Copy the REPLICATION_SCOPE only when table names and the names of + // Column-Families are same. + int result = peerHtd.getTableName().compareTo(localHtd.getTableName()); + + if (result == 0) { + Iterator remoteHCDIter = peerHtd.getFamilies().iterator(); + Iterator localHCDIter = localHtd.getFamilies().iterator(); + + while (remoteHCDIter.hasNext() && localHCDIter.hasNext()) { + HColumnDescriptor remoteHCD = remoteHCDIter.next(); + HColumnDescriptor localHCD = localHCDIter.next(); + + String remoteHCDName = remoteHCD.getNameAsString(); + String localHCDName = localHCD.getNameAsString(); + + if (remoteHCDName.equals(localHCDName)) { + remoteHCD.setScope(localHCD.getScope()); + } else { + result = -1; + break; + } + } + if (remoteHCDIter.hasNext() || localHCDIter.hasNext()) { + return false; + } + } + + return result == 0; + } + + /** + * Compare the contents of the descriptor with another one passed as a parameter for replication + * purpose. The REPLICATION_SCOPE field is ignored during comparison. + * @param peerHtd descriptor on peer cluster + * @param localHtd descriptor on source cluster which needs to be replicated. + * @return true if the contents of the two descriptors match (ignoring just REPLICATION_SCOPE). + * @see java.lang.Object#equals(java.lang.Object) + */ + private boolean compareForReplication(HTableDescriptor peerHtd, HTableDescriptor localHtd) { + if (peerHtd == localHtd) { + return true; + } + if (peerHtd == null) { + return false; + } + boolean result = false; + + // Create a copy of peer HTD as we need to change its replication + // scope to match with the local HTD. + HTableDescriptor peerHtdCopy = new HTableDescriptor(peerHtd); + + result = copyReplicationScope(peerHtdCopy, localHtd); + + // If copy was successful, compare the two tables now. + if (result) { + result = (peerHtdCopy.compareTo(localHtd) == 0); + } + + return result; + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/f34709e1/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java index ff45f11..1231173 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java @@ -51,7 +51,7 @@ import com.google.common.collect.Lists; /** * Utility class to handle the removal of HFiles (or the respective {@link StoreFile StoreFiles}) * for a HRegion from the {@link FileSystem}. The hfiles will be archived or deleted, depending on - * the state of the system. + * the state of the system. */ @InterfaceAudience.Private public class HFileArchiver { http://git-wip-us.apache.org/repos/asf/hbase/blob/f34709e1/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminWithClusters.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminWithClusters.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminWithClusters.java index b75c1cf..26ed7a7 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminWithClusters.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminWithClusters.java @@ -71,6 +71,7 @@ public class TestReplicationAdminWithClusters extends TestReplicationBase { @Test(timeout = 300000) public void testEnableReplicationWhenSlaveClusterDoesntHaveTable() throws Exception { + admin.disableTableRep(tableName); admin2.disableTable(tableName); admin2.deleteTable(tableName); assertFalse(admin2.tableExists(tableName));