Repository: hbase
Updated Branches:
refs/heads/branch-1 5f5f4e82b -> 0ec3c49c9
HBASE-21318 Make RefreshHFilesClient runnable
Signed-off-by: tedyu <yuzhihong@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0ec3c49c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0ec3c49c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0ec3c49c
Branch: refs/heads/branch-1
Commit: 0ec3c49c9df52326a39ba677671307377acaa0e8
Parents: 5f5f4e8
Author: TAK LON WU <wutaklon@amazon.com>
Authored: Fri Aug 17 15:55:10 2018 -0700
Committer: tedyu <yuzhihong@gmail.com>
Committed: Wed Oct 24 14:50:03 2018 -0700
----------------------------------------------------------------------
.../client/example/RefreshHFilesClient.java | 35 ++++++-
.../client/example/TestRefreshHFilesClient.java | 47 ++++++++++
.../example/TestRefreshHFilesBase.java | 96 ++++++++++++++++++++
.../example/TestRefreshHFilesEndpoint.java | 95 +++----------------
4 files changed, 189 insertions(+), 84 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/0ec3c49c/hbase-examples/src/main/java/org/apache/hadoop/hbase/client/example/RefreshHFilesClient.java
----------------------------------------------------------------------
diff --git a/hbase-examples/src/main/java/org/apache/hadoop/hbase/client/example/RefreshHFilesClient.java
b/hbase-examples/src/main/java/org/apache/hadoop/hbase/client/example/RefreshHFilesClient.java
index 38f0362..4334c0c 100644
--- a/hbase-examples/src/main/java/org/apache/hadoop/hbase/client/example/RefreshHFilesClient.java
+++ b/hbase-examples/src/main/java/org/apache/hadoop/hbase/client/example/RefreshHFilesClient.java
@@ -19,9 +19,13 @@
package org.apache.hadoop.hbase.client.example;
+import java.io.Closeable;
+import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Connection;
@@ -31,15 +35,14 @@ import org.apache.hadoop.hbase.client.coprocessor.Batch;
import org.apache.hadoop.hbase.ipc.BlockingRpcCallback;
import org.apache.hadoop.hbase.ipc.ServerRpcController;
import org.apache.hadoop.hbase.protobuf.generated.RefreshHFilesProtos;
-
-import java.io.Closeable;
-import java.io.IOException;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
/**
* This client class is for invoking the refresh HFile function deployed on the
* Region Server side via the RefreshHFilesService.
*/
-public class RefreshHFilesClient implements Closeable {
+public class RefreshHFilesClient extends Configured implements Tool, Closeable {
private static final Log LOG = LogFactory.getLog(RefreshHFilesClient.class);
private final Connection connection;
@@ -92,4 +95,28 @@ public class RefreshHFilesClient implements Closeable {
});
LOG.debug("Done refreshing HFiles");
}
+
+ @Override
+ public int run(String[] args) throws Exception {
+ if (args.length != 1) {
+ String message = "When there are multiple HBase clusters sharing a common root directory,
"
+ + "especially for read replica cluster (see detail in HBASE-18477), please consider
to "
+ + "use this tool manually sync the flushed HFiles from the source cluster.";
+ message += "\nUsage: " + this.getClass().getName() + " tableName";
+ System.out.println(message);
+ return -1;
+ }
+ final TableName tableName = TableName.valueOf(args[0]);
+ try {
+ refreshHFiles(tableName);
+ } catch (Throwable t) {
+ LOG.error("Refresh HFiles from table " + tableName.getNameAsString() + " failed: ",
t);
+ return -1;
+ }
+ return 0;
+ }
+
+ public static void main(String[] args) throws Exception {
+ ToolRunner.run(new RefreshHFilesClient(HBaseConfiguration.create()), args);
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0ec3c49c/hbase-examples/src/test/java/org/apache/hadoop/hbase/client/example/TestRefreshHFilesClient.java
----------------------------------------------------------------------
diff --git a/hbase-examples/src/test/java/org/apache/hadoop/hbase/client/example/TestRefreshHFilesClient.java
b/hbase-examples/src/test/java/org/apache/hadoop/hbase/client/example/TestRefreshHFilesClient.java
new file mode 100644
index 0000000..d5add30
--- /dev/null
+++ b/hbase-examples/src/test/java/org/apache/hadoop/hbase/client/example/TestRefreshHFilesClient.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client.example;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.hadoop.hbase.coprocessor.example.TestRefreshHFilesBase;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.testclassification.ClientTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.util.ToolRunner;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category({ ClientTests.class, MediumTests.class })
+public class TestRefreshHFilesClient extends TestRefreshHFilesBase {
+
+ @BeforeClass
+ public static void setUp() {
+ setUp(HRegion.class.getName());
+ }
+
+ @Test
+ public void testRefreshHFilesClient() throws Exception {
+ addHFilesToRegions();
+ assertEquals(2, HTU.getNumHFiles(TABLE_NAME, FAMILY));
+ RefreshHFilesClient tool = new RefreshHFilesClient(HTU.getConfiguration());
+ assertEquals(0, ToolRunner.run(tool, new String[] { TABLE_NAME.getNameAsString() }));
+ assertEquals(4, HTU.getNumHFiles(TABLE_NAME, FAMILY));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0ec3c49c/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesBase.java
----------------------------------------------------------------------
diff --git a/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesBase.java
b/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesBase.java
new file mode 100644
index 0000000..96ae2f8
--- /dev/null
+++ b/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesBase.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.coprocessor.example;
+
+import java.io.IOException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.apache.hadoop.hbase.regionserver.Region;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.HFileTestUtil;
+import org.junit.After;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestRefreshHFilesBase {
+ protected static final Logger LOG = LoggerFactory.getLogger(TestRefreshHFilesBase.class);
+ protected static final HBaseTestingUtility HTU = new HBaseTestingUtility();
+ protected static final TableName TABLE_NAME = TableName.valueOf("testRefreshRegionHFilesEP");
+ protected static final byte[] FAMILY = Bytes.toBytes("family");
+ protected static final byte[] QUALIFIER = Bytes.toBytes("qualifier");
+ private static final int NUM_ROWS = 5;
+ private static final int NUM_MASTER = 1;
+ private static final int NUM_RS = 2;
+ private static final byte[][] SPLIT_KEY = new byte[][] { Bytes.toBytes("30") };
+ private static final String HFILE_NAME = "123abcdef";
+
+ protected static Configuration CONF = HTU.getConfiguration();
+ private static MiniHBaseCluster cluster;
+ private static HTable table;
+ private static HTableDescriptor desc;
+ private static Admin hbaseAdmin;
+
+
+ public static void setUp(String regionImpl) {
+ try {
+ CONF.set(HConstants.REGION_IMPL, regionImpl);
+ CONF.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2);
+
+ CONF.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, RefreshHFilesEndpoint.class.getName());
+ cluster = HTU.startMiniCluster(NUM_MASTER, NUM_RS);
+
+ // Create table
+ desc = new HTableDescriptor(TABLE_NAME);
+ desc.addFamily(new HColumnDescriptor(FAMILY));
+ hbaseAdmin = cluster.getMaster().getConnection().getAdmin();
+ hbaseAdmin.createTable(desc, SPLIT_KEY);
+ table = new HTable(HTU.getConfiguration(), TABLE_NAME);
+
+ // this will create 2 regions spread across slaves
+ HTU.loadNumericRows(table, FAMILY, 1, 20);
+ HTU.flush(TABLE_NAME);
+ } catch (Exception ex) {
+ LOG.error("Couldn't finish setup", ex);
+ }
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ HTU.shutdownMiniCluster();
+ }
+
+ protected void addHFilesToRegions() throws IOException {
+ Path tableDir = desc.getTableDir(HTU.getDefaultRootDirPath(), TABLE_NAME.toBytes());
+ for (Region region : cluster.getRegions(TABLE_NAME)) {
+ Path regionDir = new Path(tableDir, region.getRegionInfo().getEncodedName());
+ Path familyDir = new Path(regionDir, Bytes.toString(FAMILY));
+ HFileTestUtil.createHFile(HTU.getConfiguration(), HTU.getTestFileSystem(),
+ new Path(familyDir, HFILE_NAME), FAMILY, QUALIFIER, Bytes.toBytes("50"),
+ Bytes.toBytes("60"), NUM_ROWS);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0ec3c49c/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesEndpoint.java
----------------------------------------------------------------------
diff --git a/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesEndpoint.java
b/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesEndpoint.java
index 9a4e293..a70eec9 100644
--- a/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesEndpoint.java
+++ b/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesEndpoint.java
@@ -15,104 +15,38 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.hadoop.hbase.coprocessor.example;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.MiniHBaseCluster;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.RetriesExhaustedException;
import org.apache.hadoop.hbase.client.example.RefreshHFilesClient;
-import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.HStore;
-import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.regionserver.RegionServerServices;
import org.apache.hadoop.hbase.regionserver.Store;
import org.apache.hadoop.hbase.testclassification.MediumTests;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.HFileTestUtil;
import org.apache.hadoop.hbase.wal.WAL;
-import org.junit.After;
import org.junit.Test;
import org.junit.experimental.categories.Category;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
@Category(MediumTests.class)
-public class TestRefreshHFilesEndpoint {
- private static final Log LOG = LogFactory.getLog(TestRefreshHFilesEndpoint.class);
- private static final HBaseTestingUtility HTU = new HBaseTestingUtility();
- private static final int NUM_MASTER = 1;
- private static final int NUM_RS = 2;
- private static final TableName TABLE_NAME = TableName.valueOf("testRefreshRegionHFilesEP");
- private static final byte[] FAMILY = Bytes.toBytes("family");
- private static final byte[] QUALIFIER = Bytes.toBytes("qualifier");
- private static final byte[][] SPLIT_KEY = new byte[][] { Bytes.toBytes("30") };
- private static final int NUM_ROWS = 5;
- private static final String HFILE_NAME = "123abcdef";
-
- private static Configuration CONF = HTU.getConfiguration();
- private static MiniHBaseCluster cluster;
- private static HTableDescriptor desc;
- private static Admin hbaseAdmin;
- private static HTable table;
-
- public static void setUp(String regionImpl) {
- try {
- CONF.set(HConstants.REGION_IMPL, regionImpl);
- CONF.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2);
-
- CONF.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, RefreshHFilesEndpoint.class.getName());
- cluster = HTU.startMiniCluster(NUM_MASTER, NUM_RS);
-
- // Create table
- desc = new HTableDescriptor(TABLE_NAME);
- desc.addFamily(new HColumnDescriptor(FAMILY));
- hbaseAdmin = cluster.getMaster().getConnection().getAdmin();
- hbaseAdmin.createTable(desc, SPLIT_KEY);
- table = new HTable(HTU.getConfiguration(), TABLE_NAME);
-
- // this will create 2 regions spread across slaves
- HTU.loadNumericRows(table, FAMILY, 1, 20);
- HTU.flush(TABLE_NAME);
- } catch (Exception ex) {
- LOG.error("Couldn't finish setup", ex);
- }
- }
-
- @After
- public void tearDown() throws Exception {
- HTU.shutdownMiniCluster();
- }
+public class TestRefreshHFilesEndpoint extends TestRefreshHFilesBase {
@Test
public void testRefreshRegionHFilesEndpoint() throws Exception {
setUp(HRegion.class.getName());
- Path tableDir = desc.getTableDir(HTU.getDefaultRootDirPath(), TABLE_NAME.toBytes());
- for (Region region : cluster.getRegions(TABLE_NAME)) {
- Path regionDir = new Path(tableDir, region.getRegionInfo().getEncodedName());
- Path familyDir = new Path(regionDir, Bytes.toString(FAMILY));
- HFileTestUtil
- .createHFile(HTU.getConfiguration(), HTU.getTestFileSystem(), new Path(familyDir,
HFILE_NAME), FAMILY,
- QUALIFIER, Bytes.toBytes("50"), Bytes.toBytes("60"), NUM_ROWS);
- }
+ addHFilesToRegions();
assertEquals(2, HTU.getNumHFiles(TABLE_NAME, FAMILY));
callRefreshRegionHFilesEndPoint();
assertEquals(4, HTU.getNumHFiles(TABLE_NAME, FAMILY));
@@ -132,7 +66,7 @@ public class TestRefreshHFilesEndpoint {
if (rex.getCause() instanceof IOException)
throw new IOException();
} catch (Throwable ex) {
- LOG.error(ex);
+ LOG.error(ex.toString(), ex);
fail("Couldn't call the RefreshRegionHFilesEndpoint");
}
}
@@ -141,8 +75,8 @@ public class TestRefreshHFilesEndpoint {
HStoreWithFaultyRefreshHFilesAPI store;
public HRegionForRefreshHFilesEP(final Path tableDir, final WAL wal, final FileSystem
fs,
- final Configuration confParam, final HRegionInfo regionInfo,
- final HTableDescriptor htd, final RegionServerServices
rsServices) {
+ final Configuration confParam, final HRegionInfo regionInfo,
+ final HTableDescriptor htd, final RegionServerServices rsServices) {
super(tableDir, wal, fs, confParam, regionInfo, htd, rsServices);
}
@@ -155,7 +89,8 @@ public class TestRefreshHFilesEndpoint {
*/
try {
if (this.store == null)
- store = new HStoreWithFaultyRefreshHFilesAPI(this, new HColumnDescriptor(FAMILY),
this.conf);
+ store = new HStoreWithFaultyRefreshHFilesAPI(this, new HColumnDescriptor(FAMILY),
+ this.conf);
list.add(store);
} catch (IOException ioe) {
LOG.info("Couldn't instantiate custom store implementation", ioe);
@@ -167,8 +102,8 @@ public class TestRefreshHFilesEndpoint {
}
public static class HStoreWithFaultyRefreshHFilesAPI extends HStore {
- public HStoreWithFaultyRefreshHFilesAPI(final HRegion region, final HColumnDescriptor
family,
- final Configuration confParam) throws IOException
{
+ public HStoreWithFaultyRefreshHFilesAPI(final HRegion region,
+ final HColumnDescriptor family, final Configuration confParam) throws IOException
{
super(region, family, confParam);
}
|