HBASE-19364, Truncate_preserve fails with table when replica region > 1
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0d8fee21
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0d8fee21
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0d8fee21
Branch: refs/heads/branch-1.4
Commit: 0d8fee2158e08bc6d0907d4abbe1215eaded6ce3
Parents: 1b0b8d9
Author: Pankaj Kumar <pankaj.kr@huawei.com>
Authored: Thu Dec 7 22:51:01 2017 +0530
Committer: Andrew Purtell <apurtell@apache.org>
Committed: Fri Feb 23 17:03:03 2018 -0800
----------------------------------------------------------------------
.../apache/hadoop/hbase/MetaTableAccessor.java | 45 +++++++++++++++++---
.../master/procedure/ProcedureSyncWait.java | 24 ++++++-----
.../procedure/TruncateTableProcedure.java | 2 +-
.../MasterProcedureTestingUtility.java | 5 +++
.../procedure/TestTruncateTableProcedure.java | 44 +++++++++++++------
5 files changed, 90 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/0d8fee21/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
index 3f11558..440f8c6 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
@@ -392,12 +392,27 @@ public class MetaTableAccessor {
* @return Ordered list of {@link HRegionInfo}.
* @throws IOException
*/
- public static List<HRegionInfo> getTableRegions(ZooKeeperWatcher zkw,
- Connection connection, TableName tableName, final boolean excludeOfflinedSplitParents)
- throws IOException {
+ public static List<HRegionInfo> getTableRegions(ZooKeeperWatcher zkw, Connection
connection,
+ TableName tableName, final boolean excludeOfflinedSplitParents) throws IOException
{
+ return getTableRegions(zkw, connection, tableName, excludeOfflinedSplitParents, false);
+ }
+
+ /**
+ * Gets all of the regions of the specified table.
+ * @param zkw zookeeper connection to access meta table
+ * @param connection connection we're using
+ * @param tableName table we're looking for
+ * @param excludeOfflinedSplitParents If true, do not include offlined split parents in
the
+ * return.
+ * @param excludeReplicaRegions If true, do not include replica regions in the result.
+ * @return Ordered list of {@link HRegionInfo}.
+ */
+ public static List<HRegionInfo> getTableRegions(ZooKeeperWatcher zkw, Connection
connection,
+ TableName tableName, final boolean excludeOfflinedSplitParents,
+ final boolean excludeReplicaRegions) throws IOException {
List<Pair<HRegionInfo, ServerName>> result = null;
- result = getTableRegionsAndLocations(zkw, connection, tableName,
- excludeOfflinedSplitParents);
+ result = getTableRegionsAndLocations(zkw, connection, tableName, excludeOfflinedSplitParents,
+ excludeReplicaRegions);
return getListOfHRegionInfos(result);
}
@@ -478,7 +493,22 @@ public class MetaTableAccessor {
public static List<Pair<HRegionInfo, ServerName>> getTableRegionsAndLocations(
ZooKeeperWatcher zkw, Connection connection, final TableName tableName,
final boolean excludeOfflinedSplitParents) throws IOException {
+ return getTableRegionsAndLocations(zkw, connection, tableName, excludeOfflinedSplitParents,
+ false);
+ }
+ /**
+ * @param zkw ZooKeeperWatcher instance we're using to get hbase:meta location
+ * @param connection connection we're using
+ * @param tableName table to work with
+ * @param excludeOfflinedSplitParents Exclude offline regions
+ * @param excludeReplicaRegions If true, do not include replica regions in the result.
+ * @return List of regioninfos and server addresses.
+ */
+ public static List<Pair<HRegionInfo, ServerName>> getTableRegionsAndLocations(
+ ZooKeeperWatcher zkw, Connection connection, final TableName tableName,
+ final boolean excludeOfflinedSplitParents, final boolean excludeReplicaRegions)
+ throws IOException {
if (tableName.equals(TableName.META_TABLE_NAME)) {
// If meta, do a bit of special handling.
ServerName serverName = new MetaTableLocator().getMetaRegionLocation(zkw);
@@ -514,6 +544,11 @@ public class MetaTableAccessor {
}
for (HRegionLocation loc : current.getRegionLocations()) {
if (loc != null) {
+ // Exclude replica region from the result
+ if (excludeReplicaRegions
+ && !RegionReplicaUtil.isDefaultReplica(loc.getRegionInfo())) {
+ continue;
+ }
this.results.add(new Pair<HRegionInfo, ServerName>(
loc.getRegionInfo(), loc.getServerName()));
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0d8fee21/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedureSyncWait.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedureSyncWait.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedureSyncWait.java
index d0dea0a..1846cef 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedureSyncWait.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedureSyncWait.java
@@ -149,22 +149,24 @@ public final class ProcedureSyncWait {
protected static List<HRegionInfo> getRegionsFromMeta(final MasterProcedureEnv env,
final TableName tableName) throws IOException {
- return getRegionsFromMeta(env, tableName, false);
+ return getRegionsFromMeta(env, tableName, false, false);
}
protected static List<HRegionInfo> getRegionsFromMeta(final MasterProcedureEnv env,
- final TableName tableName, final boolean excludeOfflinedSplitParents) throws IOException
{
+ final TableName tableName, final boolean excludeOfflinedSplitParents,
+ final boolean excludeReplicaRegions) throws IOException {
return ProcedureSyncWait.waitFor(env, "regions of table=" + tableName + " from meta",
- new ProcedureSyncWait.Predicate<List<HRegionInfo>>() {
- @Override
- public List<HRegionInfo> evaluate() throws IOException {
- if (TableName.META_TABLE_NAME.equals(tableName)) {
- return new MetaTableLocator().getMetaRegions(env.getMasterServices().getZooKeeper());
- }
- return MetaTableAccessor.getTableRegions(env.getMasterServices().getZooKeeper(),
- env.getMasterServices().getConnection(), tableName, excludeOfflinedSplitParents);
+ new ProcedureSyncWait.Predicate<List<HRegionInfo>>() {
+ @Override
+ public List<HRegionInfo> evaluate() throws IOException {
+ if (TableName.META_TABLE_NAME.equals(tableName)) {
+ return new MetaTableLocator().getMetaRegions(env.getMasterServices().getZooKeeper());
}
- });
+ return MetaTableAccessor.getTableRegions(env.getMasterServices().getZooKeeper(),
+ env.getMasterServices().getConnection(), tableName, excludeOfflinedSplitParents,
+ excludeReplicaRegions);
+ }
+ });
}
protected static void waitRegionInTransition(final MasterProcedureEnv env,
http://git-wip-us.apache.org/repos/asf/hbase/blob/0d8fee21/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
index 735e94f..120ae4f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
@@ -85,7 +85,7 @@ public class TruncateTableProcedure
// TODO: Move out... in the acquireLock()
LOG.debug("waiting for '" + getTableName() + "' regions in transition");
- regions = ProcedureSyncWait.getRegionsFromMeta(env, getTableName(), true);
+ regions = ProcedureSyncWait.getRegionsFromMeta(env, getTableName(), true, true);
assert regions != null && !regions.isEmpty() : "unexpected 0 regions";
ProcedureSyncWait.waitRegionInTransition(env, regions);
http://git-wip-us.apache.org/repos/asf/hbase/blob/0d8fee21/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
index fa57b12..ff479d4 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
@@ -74,6 +74,11 @@ public class MasterProcedureTestingUtility {
public static HRegionInfo[] createTable(final ProcedureExecutor<MasterProcedureEnv>
procExec,
final TableName tableName, final byte[][] splitKeys, String... family) throws IOException
{
HTableDescriptor htd = createHTD(tableName, family);
+ return createTable(procExec, htd, splitKeys);
+ }
+
+ public static HRegionInfo[] createTable(final ProcedureExecutor<MasterProcedureEnv>
procExec,
+ HTableDescriptor htd, final byte[][] splitKeys) throws IOException {
HRegionInfo[] regions = ModifyRegionUtils.createHRegionInfos(htd, splitKeys);
long procId = ProcedureTestingUtility.submitAndWait(procExec,
new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
http://git-wip-us.apache.org/repos/asf/hbase/blob/0d8fee21/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
index 8797a4f..47460af 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
@@ -43,6 +43,8 @@ import org.junit.experimental.categories.Category;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import java.io.IOException;
+
@Category(MediumTests.class)
public class TestTruncateTableProcedure {
private static final Log LOG = LogFactory.getLog(TestTruncateTableProcedure.class);
@@ -241,23 +243,40 @@ public class TestTruncateTableProcedure {
return UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor();
}
- @Test(timeout=60000)
- public void testTruncateWithPreserveAfterSplit() throws Exception{
+ @Test(timeout = 60000)
+ public void testTruncateWithPreserveAfterSplit() throws Exception {
final String[] families = new String[] { "f1", "f2" };
- final byte[][] splitKeys = new byte[][] {
- Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c")
- };
+ final byte[][] splitKeys =
+ new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") };
TableName tableName = TableName.valueOf("testTruncateWithPreserveAfterSplit");
- HRegionInfo[] regions = MasterProcedureTestingUtility.createTable(
- getMasterProcedureExecutor(), tableName, splitKeys, families);
+ HRegionInfo[] regions = MasterProcedureTestingUtility.createTable(getMasterProcedureExecutor(),
+ tableName, splitKeys, families);
+ splitAndTruncate(families, splitKeys, tableName, regions);
+ }
+
+ @Test(timeout = 60000)
+ public void testTruncatePreserveWithReplicaRegionAfterSplit() throws Exception {
+ final String[] families = new String[] { "f1", "f2" };
+ final byte[][] splitKeys =
+ new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") };
+ TableName tableName = TableName.valueOf("testTruncateWithPreserveAfterSplit");
+ HTableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, families);
+ htd.setRegionReplication(3);
+ HRegionInfo[] regions =
+ MasterProcedureTestingUtility.createTable(getMasterProcedureExecutor(), htd, splitKeys);
+ splitAndTruncate(families, splitKeys, tableName, regions);
+ }
+
+ private void splitAndTruncate(final String[] families, final byte[][] splitKeys,
+ TableName tableName, HRegionInfo[] regions) throws IOException, InterruptedException
{
// load enough data so the table can split
- MasterProcedureTestingUtility.loadData(
- UTIL.getConnection(), tableName, 5000, splitKeys, families);
+ MasterProcedureTestingUtility.loadData(UTIL.getConnection(), tableName, 5000, splitKeys,
+ families);
assertEquals(5000, UTIL.countRows(tableName));
UTIL.getHBaseAdmin().split(tableName);
UTIL.waitUntilAllRegionsAssigned(tableName);
- //wait until split really happens
- while(UTIL.getHBaseAdmin().getTableRegions(tableName).size() <= regions.length) {
+ // wait until split really happens
+ while (UTIL.getHBaseAdmin().getTableRegions(tableName).size() <= regions.length) {
Thread.sleep(50);
}
// disable the table
@@ -265,10 +284,9 @@ public class TestTruncateTableProcedure {
// truncate the table
final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
long procId = ProcedureTestingUtility.submitAndWait(procExec,
- new TruncateTableProcedure(procExec.getEnvironment(), tableName, true));
+ new TruncateTableProcedure(procExec.getEnvironment(), tableName, true));
ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
UTIL.waitUntilAllRegionsAssigned(tableName);
-
}
}
|