Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id CE224200B78 for ; Fri, 2 Sep 2016 19:43:30 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id CC8CC160ACB; Fri, 2 Sep 2016 17:43:30 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id BDA86160AAE for ; Fri, 2 Sep 2016 19:43:29 +0200 (CEST) Received: (qmail 71954 invoked by uid 500); 2 Sep 2016 17:43:27 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 71895 invoked by uid 99); 2 Sep 2016 17:43:27 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 02 Sep 2016 17:43:27 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 7245DE0061; Fri, 2 Sep 2016 17:43:27 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: apurtell@apache.org To: commits@hbase.apache.org Date: Fri, 02 Sep 2016 17:43:28 -0000 Message-Id: <3636453faa024c35b8069e984180acce@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [2/2] hbase git commit: HBASE-16399 Provide an API to get list of failed regions and servername in Canary (Vishal Khandelwal) archived-at: Fri, 02 Sep 2016 17:43:31 -0000 HBASE-16399 Provide an API to get list of failed regions and servername in Canary (Vishal Khandelwal) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0ac78f44 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0ac78f44 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0ac78f44 Branch: refs/heads/0.98 Commit: 0ac78f440cdfc233711a5eb8a3db58db1a837d37 Parents: 51a5fe8 Author: Andrew Purtell Authored: Fri Sep 2 10:13:35 2016 -0700 Committer: Andrew Purtell Committed: Fri Sep 2 10:37:22 2016 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/hbase/tool/Canary.java | 53 ++++++++++++++++++-- .../hadoop/hbase/tool/TestCanaryTool.java | 12 +++-- 2 files changed, 56 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/0ac78f44/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java index 51f162f..d887e4d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java @@ -40,6 +40,7 @@ import java.util.Random; import java.util.Set; import java.util.TreeSet; import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; @@ -113,11 +114,15 @@ public final class Canary implements Tool { public long incReadFailureCount(); public void publishReadFailure(HRegionInfo region, Exception e); public void publishReadFailure(HRegionInfo region, HColumnDescriptor column, Exception e); + public void updateReadFailedHostList(HRegionInfo region, String serverName); + public Map getReadFailures(); public void publishReadTiming(HRegionInfo region, HColumnDescriptor column, long msTime); public long getWriteFailureCount(); public void publishWriteFailure(HRegionInfo region, Exception e); public void publishWriteFailure(HRegionInfo region, HColumnDescriptor column, Exception e); public void publishWriteTiming(HRegionInfo region, HColumnDescriptor column, long msTime); + public void updateWriteFailedHostList(HRegionInfo region, String serverName); + public Map getWriteFailures(); } // new extended sink for output regionserver mode info // do not change the Sink interface directly due to maintaining the API @@ -132,6 +137,9 @@ public final class Canary implements Tool { private AtomicLong readFailureCount = new AtomicLong(0), writeFailureCount = new AtomicLong(0); + private Map readFailures = new ConcurrentHashMap(); + private Map writeFailures = new ConcurrentHashMap(); + @Override public long getReadFailureCount() { return readFailureCount.get(); @@ -156,9 +164,24 @@ public final class Canary implements Tool { } @Override + public void updateReadFailedHostList(HRegionInfo region, String serverName) { + readFailures.put(region.getRegionNameAsString(), serverName); + } + + @Override public void publishReadTiming(HRegionInfo region, HColumnDescriptor column, long msTime) { LOG.info(String.format("read from region %s column family %s in %dms", - region.getRegionNameAsString(), column.getNameAsString(), msTime)); + region.getRegionNameAsString(), column.getNameAsString(), msTime)); + } + + @Override + public Map getReadFailures() { + return readFailures; + } + + @Override + public Map getWriteFailures() { + return writeFailures; } @Override @@ -184,6 +207,12 @@ public final class Canary implements Tool { LOG.info(String.format("write to region %s column family %s in %dms", region.getRegionNameAsString(), column.getNameAsString(), msTime)); } + + @Override + public void updateWriteFailedHostList(HRegionInfo region, String serverName) { + writeFailures.put(region.getRegionNameAsString(), serverName); + } + } // a ExtendedSink implementation public static class RegionServerStdOutSink extends StdOutSink implements ExtendedSink { @@ -265,14 +294,16 @@ public final class Canary implements Tool { private Sink sink; private TaskType taskType; private boolean rawScanEnabled; + private ServerName serverName; - RegionTask(HConnection connection, HRegionInfo region, Sink sink, TaskType taskType, - boolean rawScanEnabled) { + RegionTask(HConnection connection, HRegionInfo region, ServerName serverName, Sink sink, + TaskType taskType, boolean rawScanEnabled) { this.connection = connection; this.region = region; this.sink = sink; this.taskType = taskType; this.rawScanEnabled = rawScanEnabled; + this.serverName = serverName; } @Override @@ -356,6 +387,7 @@ public final class Canary implements Tool { sink.publishReadTiming(region, column, stopWatch.getTime()); } catch (Exception e) { sink.publishReadFailure(region, column, e); + sink.updateReadFailedHostList(region, serverName.getHostname()); } finally { if (rs != null) { rs.close(); @@ -412,6 +444,7 @@ public final class Canary implements Tool { table.close(); } catch (IOException e) { sink.publishWriteFailure(region, e); + sink.updateWriteFailedHostList(region, serverName.getHostname()); } return null; } @@ -483,9 +516,11 @@ public final class Canary implements Tool { LOG.debug("The targeted table was disabled. Assuming success."); } catch (DoNotRetryIOException dnrioe) { sink.publishReadFailure(tableName.getNameAsString(), serverName); + sink.updateReadFailedHostList(region, serverName); LOG.error(dnrioe); } catch (IOException e) { sink.publishReadFailure(tableName.getNameAsString(), serverName); + sink.updateReadFailedHostList(region, serverName); LOG.error(e); } finally { if (table != null) { @@ -720,6 +755,14 @@ public final class Canary implements Tool { return monitor.errorCode; } + public Map getReadFailures() { + return sink.getReadFailures(); + } + + public Map getWriteFailures() { + return sink.getWriteFailures(); + } + private void printUsageAndExit() { System.err.printf( "Usage: bin/hbase %s [opts] [table1 [table2]...] | [regionserver1 [regionserver2]..]%n", @@ -1103,8 +1146,8 @@ public final class Canary implements Tool { } List tasks = new ArrayList(); try { - for (HRegionInfo region : ((HTable)table).getRegionLocations().keySet()) { - tasks.add(new RegionTask(connection, region, sink, taskType, rawScanEnabled)); + for (Map.Entry region : ((HTable)table).getRegionLocations().entrySet()) { + tasks.add(new RegionTask(connection, region.getKey(), region.getValue(), sink, taskType, rawScanEnabled)); } } finally { table.close(); http://git-wip-us.apache.org/repos/asf/hbase/blob/0ac78f44/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java index 9d1dd9e..877eb66 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java @@ -52,6 +52,7 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Matchers.argThat; import static org.mockito.Mockito.never; +import static org.junit.Assert.assertEquals; @RunWith(MockitoJUnitRunner.class) @Category({MediumTests.class}) @@ -108,10 +109,11 @@ public class TestCanaryTool { ExecutorService executor = new ScheduledThreadPoolExecutor(1); Canary.RegionServerStdOutSink sink = spy(new Canary.RegionServerStdOutSink()); Canary canary = new Canary(executor, sink); - String[] args = { "-t", "10000", "testTable" }; + String[] args = { "-writeSniffing", "-t", "10000", "testTable" }; ToolRunner.run(testingUtility.getConfiguration(), canary, args); - verify(sink, atLeastOnce()) - .publishReadTiming(isA(HRegionInfo.class), isA(HColumnDescriptor.class), anyLong()); + assertEquals("verify no read error count", 0, canary.getReadFailures().size()); + assertEquals("verify no write error count", 0, canary.getWriteFailures().size()); + verify(sink, atLeastOnce()).publishReadTiming(isA(HRegionInfo.class), isA(HColumnDescriptor.class), anyLong()); } //no table created, so there should be no regions @@ -160,13 +162,15 @@ public class TestCanaryTool { ToolRunner.run(conf, canary, args); verify(sink, atLeastOnce()) .publishReadTiming(isA(HRegionInfo.class), isA(HColumnDescriptor.class), anyLong()); + assertEquals("verify no read error count", 0, canary.getReadFailures().size()); } - + private void runRegionserverCanary() throws Exception { ExecutorService executor = new ScheduledThreadPoolExecutor(1); Canary canary = new Canary(executor, new Canary.RegionServerStdOutSink()); String[] args = { "-t", "10000", "-regionserver"}; ToolRunner.run(testingUtility.getConfiguration(), canary, args); + assertEquals("verify no read error count", 0, canary.getReadFailures().size()); } }