hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From te...@apache.org
Subject hbase git commit: HBASE-21318 Make RefreshHFilesClient runnable
Date Wed, 24 Oct 2018 21:50:08 GMT
Repository: hbase
Updated Branches:
  refs/heads/branch-1 5f5f4e82b -> 0ec3c49c9


HBASE-21318 Make RefreshHFilesClient runnable

Signed-off-by: tedyu <yuzhihong@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0ec3c49c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0ec3c49c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0ec3c49c

Branch: refs/heads/branch-1
Commit: 0ec3c49c9df52326a39ba677671307377acaa0e8
Parents: 5f5f4e8
Author: TAK LON WU <wutaklon@amazon.com>
Authored: Fri Aug 17 15:55:10 2018 -0700
Committer: tedyu <yuzhihong@gmail.com>
Committed: Wed Oct 24 14:50:03 2018 -0700

----------------------------------------------------------------------
 .../client/example/RefreshHFilesClient.java     | 35 ++++++-
 .../client/example/TestRefreshHFilesClient.java | 47 ++++++++++
 .../example/TestRefreshHFilesBase.java          | 96 ++++++++++++++++++++
 .../example/TestRefreshHFilesEndpoint.java      | 95 +++----------------
 4 files changed, 189 insertions(+), 84 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/0ec3c49c/hbase-examples/src/main/java/org/apache/hadoop/hbase/client/example/RefreshHFilesClient.java
----------------------------------------------------------------------
diff --git a/hbase-examples/src/main/java/org/apache/hadoop/hbase/client/example/RefreshHFilesClient.java
b/hbase-examples/src/main/java/org/apache/hadoop/hbase/client/example/RefreshHFilesClient.java
index 38f0362..4334c0c 100644
--- a/hbase-examples/src/main/java/org/apache/hadoop/hbase/client/example/RefreshHFilesClient.java
+++ b/hbase-examples/src/main/java/org/apache/hadoop/hbase/client/example/RefreshHFilesClient.java
@@ -19,9 +19,13 @@
 
 package org.apache.hadoop.hbase.client.example;
 
+import java.io.Closeable;
+import java.io.IOException;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.Connection;
@@ -31,15 +35,14 @@ import org.apache.hadoop.hbase.client.coprocessor.Batch;
 import org.apache.hadoop.hbase.ipc.BlockingRpcCallback;
 import org.apache.hadoop.hbase.ipc.ServerRpcController;
 import org.apache.hadoop.hbase.protobuf.generated.RefreshHFilesProtos;
-
-import java.io.Closeable;
-import java.io.IOException;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
 
 /**
  * This client class is for invoking the refresh HFile function deployed on the
  * Region Server side via the RefreshHFilesService.
  */
-public class RefreshHFilesClient implements Closeable {
+public class RefreshHFilesClient extends Configured implements Tool, Closeable {
   private static final Log LOG = LogFactory.getLog(RefreshHFilesClient.class);
   private final Connection connection;
 
@@ -92,4 +95,28 @@ public class RefreshHFilesClient implements Closeable {
                              });
     LOG.debug("Done refreshing HFiles");
   }
+
+  @Override
+  public int run(String[] args) throws Exception {
+    if (args.length != 1) {
+      String message = "When there are multiple HBase clusters sharing a common root directory,
"
+          + "especially for read replica cluster (see detail in HBASE-18477), please consider
to "
+          + "use this tool manually sync the flushed HFiles from the source cluster.";
+      message += "\nUsage: " + this.getClass().getName() + " tableName";
+      System.out.println(message);
+      return -1;
+    }
+    final TableName tableName = TableName.valueOf(args[0]);
+    try {
+      refreshHFiles(tableName);
+    } catch (Throwable t) {
+      LOG.error("Refresh HFiles from table " + tableName.getNameAsString() + "  failed: ",
t);
+      return -1;
+    }
+    return 0;
+  }
+
+  public static void main(String[] args) throws Exception {
+    ToolRunner.run(new RefreshHFilesClient(HBaseConfiguration.create()), args);
+  }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/0ec3c49c/hbase-examples/src/test/java/org/apache/hadoop/hbase/client/example/TestRefreshHFilesClient.java
----------------------------------------------------------------------
diff --git a/hbase-examples/src/test/java/org/apache/hadoop/hbase/client/example/TestRefreshHFilesClient.java
b/hbase-examples/src/test/java/org/apache/hadoop/hbase/client/example/TestRefreshHFilesClient.java
new file mode 100644
index 0000000..d5add30
--- /dev/null
+++ b/hbase-examples/src/test/java/org/apache/hadoop/hbase/client/example/TestRefreshHFilesClient.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client.example;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.hadoop.hbase.coprocessor.example.TestRefreshHFilesBase;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.testclassification.ClientTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.util.ToolRunner;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category({ ClientTests.class, MediumTests.class })
+public class TestRefreshHFilesClient extends TestRefreshHFilesBase {
+
+  @BeforeClass
+  public static void setUp() {
+    setUp(HRegion.class.getName());
+  }
+
+  @Test
+  public void testRefreshHFilesClient() throws Exception {
+    addHFilesToRegions();
+    assertEquals(2, HTU.getNumHFiles(TABLE_NAME, FAMILY));
+    RefreshHFilesClient tool = new RefreshHFilesClient(HTU.getConfiguration());
+    assertEquals(0, ToolRunner.run(tool, new String[] { TABLE_NAME.getNameAsString() }));
+    assertEquals(4, HTU.getNumHFiles(TABLE_NAME, FAMILY));
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/0ec3c49c/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesBase.java
----------------------------------------------------------------------
diff --git a/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesBase.java
b/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesBase.java
new file mode 100644
index 0000000..96ae2f8
--- /dev/null
+++ b/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesBase.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.coprocessor.example;
+
+import java.io.IOException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.apache.hadoop.hbase.regionserver.Region;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.HFileTestUtil;
+import org.junit.After;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestRefreshHFilesBase {
+  protected static final Logger LOG = LoggerFactory.getLogger(TestRefreshHFilesBase.class);
+  protected static final HBaseTestingUtility HTU = new HBaseTestingUtility();
+  protected static final TableName TABLE_NAME = TableName.valueOf("testRefreshRegionHFilesEP");
+  protected static final byte[] FAMILY = Bytes.toBytes("family");
+  protected static final byte[] QUALIFIER = Bytes.toBytes("qualifier");
+  private static final int NUM_ROWS = 5;
+  private static final int NUM_MASTER = 1;
+  private static final int NUM_RS = 2;
+  private static final byte[][] SPLIT_KEY = new byte[][] { Bytes.toBytes("30") };
+  private static final String HFILE_NAME = "123abcdef";
+
+  protected static Configuration CONF = HTU.getConfiguration();
+  private static MiniHBaseCluster cluster;
+  private static HTable table;
+  private static HTableDescriptor desc;
+  private static Admin hbaseAdmin;
+
+
+  public static void setUp(String regionImpl) {
+    try {
+      CONF.set(HConstants.REGION_IMPL, regionImpl);
+      CONF.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2);
+
+      CONF.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, RefreshHFilesEndpoint.class.getName());
+      cluster = HTU.startMiniCluster(NUM_MASTER, NUM_RS);
+
+      // Create table
+      desc = new HTableDescriptor(TABLE_NAME);
+      desc.addFamily(new HColumnDescriptor(FAMILY));
+      hbaseAdmin = cluster.getMaster().getConnection().getAdmin();
+      hbaseAdmin.createTable(desc, SPLIT_KEY);
+      table = new HTable(HTU.getConfiguration(), TABLE_NAME);
+
+      // this will create 2 regions spread across slaves
+      HTU.loadNumericRows(table, FAMILY, 1, 20);
+      HTU.flush(TABLE_NAME);
+    } catch (Exception ex) {
+      LOG.error("Couldn't finish setup", ex);
+    }
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    HTU.shutdownMiniCluster();
+  }
+
+  protected void addHFilesToRegions() throws IOException {
+    Path tableDir = desc.getTableDir(HTU.getDefaultRootDirPath(), TABLE_NAME.toBytes());
+    for (Region region : cluster.getRegions(TABLE_NAME)) {
+      Path regionDir = new Path(tableDir, region.getRegionInfo().getEncodedName());
+      Path familyDir = new Path(regionDir, Bytes.toString(FAMILY));
+      HFileTestUtil.createHFile(HTU.getConfiguration(), HTU.getTestFileSystem(),
+          new Path(familyDir, HFILE_NAME), FAMILY, QUALIFIER, Bytes.toBytes("50"),
+          Bytes.toBytes("60"), NUM_ROWS);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/0ec3c49c/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesEndpoint.java
----------------------------------------------------------------------
diff --git a/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesEndpoint.java
b/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesEndpoint.java
index 9a4e293..a70eec9 100644
--- a/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesEndpoint.java
+++ b/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesEndpoint.java
@@ -15,104 +15,38 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 package org.apache.hadoop.hbase.coprocessor.example;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.MiniHBaseCluster;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.RetriesExhaustedException;
 import org.apache.hadoop.hbase.client.example.RefreshHFilesClient;
-import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.HStore;
-import org.apache.hadoop.hbase.regionserver.Region;
 import org.apache.hadoop.hbase.regionserver.RegionServerServices;
 import org.apache.hadoop.hbase.regionserver.Store;
 import org.apache.hadoop.hbase.testclassification.MediumTests;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.HFileTestUtil;
 import org.apache.hadoop.hbase.wal.WAL;
-import org.junit.After;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
 @Category(MediumTests.class)
-public class TestRefreshHFilesEndpoint {
-  private static final Log LOG = LogFactory.getLog(TestRefreshHFilesEndpoint.class);
-  private static final HBaseTestingUtility HTU = new HBaseTestingUtility();
-  private static final int NUM_MASTER = 1;
-  private static final int NUM_RS = 2;
-  private static final TableName TABLE_NAME = TableName.valueOf("testRefreshRegionHFilesEP");
-  private static final byte[] FAMILY = Bytes.toBytes("family");
-  private static final byte[] QUALIFIER = Bytes.toBytes("qualifier");
-  private static final byte[][] SPLIT_KEY = new byte[][] { Bytes.toBytes("30") };
-  private static final int NUM_ROWS = 5;
-  private static final String HFILE_NAME = "123abcdef";
-
-  private static Configuration CONF = HTU.getConfiguration();
-  private static MiniHBaseCluster cluster;
-  private static HTableDescriptor desc;
-  private static Admin hbaseAdmin;
-  private static HTable table;
-
-  public static void setUp(String regionImpl) {
-    try {
-      CONF.set(HConstants.REGION_IMPL, regionImpl);
-      CONF.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2);
-
-      CONF.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, RefreshHFilesEndpoint.class.getName());
-      cluster = HTU.startMiniCluster(NUM_MASTER, NUM_RS);
-
-      // Create table
-      desc = new HTableDescriptor(TABLE_NAME);
-      desc.addFamily(new HColumnDescriptor(FAMILY));
-      hbaseAdmin = cluster.getMaster().getConnection().getAdmin();
-      hbaseAdmin.createTable(desc, SPLIT_KEY);
-      table = new HTable(HTU.getConfiguration(), TABLE_NAME);
-
-      // this will create 2 regions spread across slaves
-      HTU.loadNumericRows(table, FAMILY, 1, 20);
-      HTU.flush(TABLE_NAME);
-    } catch (Exception ex) {
-      LOG.error("Couldn't finish setup", ex);
-    }
-  }
-
-  @After
-  public void tearDown() throws Exception {
-    HTU.shutdownMiniCluster();
-  }
+public class TestRefreshHFilesEndpoint extends TestRefreshHFilesBase {
 
   @Test
   public void testRefreshRegionHFilesEndpoint() throws Exception {
     setUp(HRegion.class.getName());
-    Path tableDir = desc.getTableDir(HTU.getDefaultRootDirPath(), TABLE_NAME.toBytes());
-    for (Region region : cluster.getRegions(TABLE_NAME)) {
-      Path regionDir = new Path(tableDir, region.getRegionInfo().getEncodedName());
-      Path familyDir = new Path(regionDir, Bytes.toString(FAMILY));
-      HFileTestUtil
-        .createHFile(HTU.getConfiguration(), HTU.getTestFileSystem(), new Path(familyDir,
HFILE_NAME), FAMILY,
-                     QUALIFIER, Bytes.toBytes("50"), Bytes.toBytes("60"), NUM_ROWS);
-    }
+    addHFilesToRegions();
     assertEquals(2, HTU.getNumHFiles(TABLE_NAME, FAMILY));
     callRefreshRegionHFilesEndPoint();
     assertEquals(4, HTU.getNumHFiles(TABLE_NAME, FAMILY));
@@ -132,7 +66,7 @@ public class TestRefreshHFilesEndpoint {
       if (rex.getCause() instanceof IOException)
         throw new IOException();
     } catch (Throwable ex) {
-      LOG.error(ex);
+      LOG.error(ex.toString(), ex);
       fail("Couldn't call the RefreshRegionHFilesEndpoint");
     }
   }
@@ -141,8 +75,8 @@ public class TestRefreshHFilesEndpoint {
     HStoreWithFaultyRefreshHFilesAPI store;
 
     public HRegionForRefreshHFilesEP(final Path tableDir, final WAL wal, final FileSystem
fs,
-                                     final Configuration confParam, final HRegionInfo regionInfo,
-                                     final HTableDescriptor htd, final RegionServerServices
rsServices) {
+        final Configuration confParam, final HRegionInfo regionInfo,
+        final HTableDescriptor htd, final RegionServerServices rsServices) {
       super(tableDir, wal, fs, confParam, regionInfo, htd, rsServices);
     }
 
@@ -155,7 +89,8 @@ public class TestRefreshHFilesEndpoint {
        */
       try {
         if (this.store == null)
-          store = new HStoreWithFaultyRefreshHFilesAPI(this, new HColumnDescriptor(FAMILY),
this.conf);
+          store = new HStoreWithFaultyRefreshHFilesAPI(this, new HColumnDescriptor(FAMILY),
+              this.conf);
         list.add(store);
       } catch (IOException ioe) {
         LOG.info("Couldn't instantiate custom store implementation", ioe);
@@ -167,8 +102,8 @@ public class TestRefreshHFilesEndpoint {
   }
 
   public static class HStoreWithFaultyRefreshHFilesAPI extends HStore {
-    public HStoreWithFaultyRefreshHFilesAPI(final HRegion region, final HColumnDescriptor
family,
-                                            final Configuration confParam) throws IOException
{
+    public HStoreWithFaultyRefreshHFilesAPI(final HRegion region,
+        final HColumnDescriptor family, final Configuration confParam) throws IOException
{
       super(region, family, confParam);
     }
 


Mime
View raw message