Return-Path: X-Original-To: apmail-hbase-commits-archive@www.apache.org Delivered-To: apmail-hbase-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 0FCC817E78 for ; Fri, 1 May 2015 23:33:31 +0000 (UTC) Received: (qmail 31529 invoked by uid 500); 1 May 2015 23:33:30 -0000 Delivered-To: apmail-hbase-commits-archive@hbase.apache.org Received: (qmail 31472 invoked by uid 500); 1 May 2015 23:33:30 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 31461 invoked by uid 99); 1 May 2015 23:33:30 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 01 May 2015 23:33:30 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id BE034E179E; Fri, 1 May 2015 23:33:30 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: apurtell@apache.org To: commits@hbase.apache.org Date: Fri, 01 May 2015 23:33:30 -0000 Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: [1/5] hbase git commit: HBASE-13312 SmallScannerCallable does not increment scan metrics Repository: hbase Updated Branches: refs/heads/0.98 4a82713f6 -> 97efb9215 refs/heads/branch-1 3d722dce5 -> 60b356a85 refs/heads/branch-1.0 099fff5ba -> 7a2e77244 refs/heads/branch-1.1 884b0a5c8 -> bbcc9c2bb refs/heads/master 58689b4a0 -> a6027aedb HBASE-13312 SmallScannerCallable does not increment scan metrics Conflicts: hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientSmallScanner.java Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/97efb921 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/97efb921 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/97efb921 Branch: refs/heads/0.98 Commit: 97efb92155a6f256c5ed2544bb79302eb6653747 Parents: 4a82713 Author: Andrew Purtell Authored: Fri May 1 16:33:09 2015 -0700 Committer: Andrew Purtell Committed: Fri May 1 16:33:09 2015 -0700 ---------------------------------------------------------------------- .../client/ClientSmallReversedScanner.java | 3 +- .../hadoop/hbase/client/ClientSmallScanner.java | 68 ++++++++++++-------- .../hadoop/hbase/client/ScannerCallable.java | 4 +- .../client/TestClientSmallReversedScanner.java | 3 +- .../hbase/client/TestClientSmallScanner.java | 3 +- .../hadoop/hbase/client/TestFromClientSide.java | 34 ++++++++++ 6 files changed, 84 insertions(+), 31 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/97efb921/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientSmallReversedScanner.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientSmallReversedScanner.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientSmallReversedScanner.java index 21491ad..c0e3f1f 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientSmallReversedScanner.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientSmallReversedScanner.java @@ -138,7 +138,8 @@ public class ClientSmallReversedScanner extends ReversedClientScanner { } smallScanCallable = callableFactory.getCallable( - scan, getConnection(), getTable(), localStartKey, cacheNum, this.rpcControllerFactory); + scan, getConnection(), getTable(), scanMetrics, localStartKey, cacheNum, + rpcControllerFactory); if (this.scanMetrics != null && regionChanged) { this.scanMetrics.countOfRegions.incrementAndGet(); http://git-wip-us.apache.org/repos/asf/hbase/blob/97efb921/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientSmallScanner.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientSmallScanner.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientSmallScanner.java index 1ff7788..e6c2c8e 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientSmallScanner.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientSmallScanner.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.client; import java.io.IOException; +import java.io.InterruptedIOException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -29,6 +30,7 @@ import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.client.metrics.ScanMetrics; import org.apache.hadoop.hbase.ipc.PayloadCarryingRpcController; import org.apache.hadoop.hbase.ipc.RpcControllerFactory; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; @@ -184,13 +186,48 @@ public class ClientSmallScanner extends ClientScanner { + Bytes.toStringBinary(localStartKey) + "'"); } smallScanCallable = callableFactory.getCallable( - scan, getConnection(), getTable(), localStartKey, cacheNum, rpcControllerFactory); + scan, getConnection(), getTable(), scanMetrics, localStartKey, cacheNum, + rpcControllerFactory); if (this.scanMetrics != null && regionChanged) { this.scanMetrics.countOfRegions.incrementAndGet(); } return true; } + static class SmallScannerCallable extends ScannerCallable { + public SmallScannerCallable(HConnection connection, TableName tableName, Scan scan, + ScanMetrics scanMetrics, PayloadCarryingRpcController controller, int cacheNum) { + super(connection, tableName, scan, scanMetrics, controller); + this.setCaching(cacheNum); + } + + @Override + public Result [] call() throws IOException { + if (this.closed) return null; + if (Thread.interrupted()) { + throw new InterruptedIOException(); + } + ScanRequest request = RequestConverter.buildScanRequest(getLocation() + .getRegionInfo().getRegionName(), getScan(), getCaching(), true); + ScanResponse response = null; + try { + controller.setPriority(getTableName()); + response = getStub().scan(controller, request); + Result[] results = ResponseConverter.getResults(controller.cellScanner(), response); + if (response.hasMoreResultsInRegion()) { + setHasMoreResultsContext(true); + setServerHasMoreResults(response.getMoreResultsInRegion()); + } else { + setHasMoreResultsContext(false); + } + updateResultsMetrics(results); + return results; + } catch (ServiceException se) { + throw ProtobufUtil.getRemoteException(se); + } + } + } + @Override public Result next() throws IOException { // If the scanner is closed and there's nothing left in the cache, next is a @@ -276,33 +313,12 @@ public class ClientSmallScanner extends ClientScanner { protected static class SmallScannerCallableFactory { public RegionServerCallable getCallable(final Scan sc, HConnection connection, - TableName table, byte[] localStartKey, final int cacheNum, + TableName tableName, ScanMetrics scanMetrics, byte[] localStartKey, final int cacheNum, final RpcControllerFactory rpcControllerFactory) throws IOException { sc.setStartRow(localStartKey); - RegionServerCallable callable = new RegionServerCallable( - connection, table, sc.getStartRow()) { - public Result[] call() throws IOException { - ScanRequest request = RequestConverter.buildScanRequest(getLocation() - .getRegionInfo().getRegionName(), sc, cacheNum, true); - ScanResponse response = null; - PayloadCarryingRpcController controller = rpcControllerFactory.newController(); - try { - controller.setPriority(getTableName()); - response = getStub().scan(controller, request); - if (response.hasMoreResultsInRegion()) { - setHasMoreResultsContext(true); - setServerHasMoreResults(response.getMoreResultsInRegion()); - } else { - setHasMoreResultsContext(false); - } - return ResponseConverter.getResults(controller.cellScanner(), - response); - } catch (ServiceException se) { - throw ProtobufUtil.getRemoteException(se); - } - } - }; - return callable; + PayloadCarryingRpcController controller = rpcControllerFactory.newController(); + return new SmallScannerCallable(connection, tableName, sc, scanMetrics, controller, + cacheNum); } } http://git-wip-us.apache.org/repos/asf/hbase/blob/97efb921/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java index cba12a8..f8da289 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java @@ -64,7 +64,7 @@ public class ScannerCallable extends RegionServerCallable { public static final Log LOG = LogFactory.getLog(ScannerCallable.class); private long scannerId = -1L; protected boolean instantiated = false; - private boolean closed = false; + protected boolean closed = false; private Scan scan; private int caching = 1; protected ScanMetrics scanMetrics; @@ -272,7 +272,7 @@ public class ScannerCallable extends RegionServerCallable { } } - private void updateResultsMetrics(Result[] rrs) { + protected void updateResultsMetrics(Result[] rrs) { if (this.scanMetrics == null || rrs == null || rrs.length == 0) { return; } http://git-wip-us.apache.org/repos/asf/hbase/blob/97efb921/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientSmallReversedScanner.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientSmallReversedScanner.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientSmallReversedScanner.java index 84932d0..061d6ce 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientSmallReversedScanner.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientSmallReversedScanner.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValue.Type; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.ClientSmallScanner.SmallScannerCallableFactory; +import org.apache.hadoop.hbase.client.metrics.ScanMetrics; import org.apache.hadoop.hbase.ipc.RpcControllerFactory; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.junit.After; @@ -105,7 +106,7 @@ public class TestClientSmallReversedScanner { return new SmallScannerCallableFactory() { @Override public RegionServerCallable getCallable(final Scan sc, HConnection connection, - TableName table, byte[] localStartKey, final int cacheNum, + TableName table, ScanMetrics scanMetrics, byte[] localStartKey, final int cacheNum, final RpcControllerFactory rpcControllerFactory) { return callableWithReplicas; } http://git-wip-us.apache.org/repos/asf/hbase/blob/97efb921/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientSmallScanner.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientSmallScanner.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientSmallScanner.java index 971020b..debdc59 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientSmallScanner.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientSmallScanner.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValue.Type; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.ClientSmallScanner.SmallScannerCallableFactory; +import org.apache.hadoop.hbase.client.metrics.ScanMetrics; import org.apache.hadoop.hbase.ipc.RpcControllerFactory; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.junit.After; @@ -104,7 +105,7 @@ public class TestClientSmallScanner { return new SmallScannerCallableFactory() { @Override public RegionServerCallable getCallable(final Scan sc, HConnection connection, - TableName table, byte[] localStartKey, final int cacheNum, + TableName table, ScanMetrics scanMetrics, byte[] localStartKey, final int cacheNum, final RpcControllerFactory rpcControllerFactory) { return callableWithReplicas; } http://git-wip-us.apache.org/repos/asf/hbase/blob/97efb921/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java index ddb713f..b543434 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java @@ -59,6 +59,7 @@ import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.MiniHBaseCluster; import org.apache.hadoop.hbase.ServerName; @@ -5044,6 +5045,39 @@ public class TestFromClientSide { assertEquals("Did not access all the regions in the table", numOfRegions, scanMetrics.countOfRegions.get()); + // check byte counters + scan = new Scan(); + scan.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE, Bytes.toBytes(Boolean.TRUE)); + scan.setCaching(1); + scanner = ht.getScanner(scan); + int numBytes = 0; + for (Result result : scanner.next(1)) { + for (Cell cell: result.listCells()) { + numBytes += KeyValueUtil.ensureKeyValue(cell).getLength(); + } + } + scanner.close(); + scanMetrics = getScanMetrics(scan); + assertEquals("Did not count the result bytes", numBytes, + scanMetrics.countOfBytesInResults.get()); + + // check byte counters on a small scan + scan = new Scan(); + scan.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE, Bytes.toBytes(Boolean.TRUE)); + scan.setCaching(1); + scan.setSmall(true); + scanner = ht.getScanner(scan); + numBytes = 0; + for (Result result : scanner.next(1)) { + for (Cell cell: result.listCells()) { + numBytes += KeyValueUtil.ensureKeyValue(cell).getLength(); + } + } + scanner.close(); + scanMetrics = getScanMetrics(scan); + assertEquals("Did not count the result bytes", numBytes, + scanMetrics.countOfBytesInResults.get()); + // now, test that the metrics are still collected even if you don't call close, but do // run past the end of all the records Scan scanWithoutClose = new Scan();