hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From apurt...@apache.org
Subject [1/5] hbase git commit: HBASE-13312 SmallScannerCallable does not increment scan metrics
Date Fri, 01 May 2015 23:33:30 GMT
Repository: hbase
Updated Branches:
  refs/heads/0.98 4a82713f6 -> 97efb9215
  refs/heads/branch-1 3d722dce5 -> 60b356a85
  refs/heads/branch-1.0 099fff5ba -> 7a2e77244
  refs/heads/branch-1.1 884b0a5c8 -> bbcc9c2bb
  refs/heads/master 58689b4a0 -> a6027aedb


HBASE-13312 SmallScannerCallable does not increment scan metrics

Conflicts:
	hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientSmallScanner.java


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/97efb921
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/97efb921
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/97efb921

Branch: refs/heads/0.98
Commit: 97efb92155a6f256c5ed2544bb79302eb6653747
Parents: 4a82713
Author: Andrew Purtell <apurtell@apache.org>
Authored: Fri May 1 16:33:09 2015 -0700
Committer: Andrew Purtell <apurtell@apache.org>
Committed: Fri May 1 16:33:09 2015 -0700

----------------------------------------------------------------------
 .../client/ClientSmallReversedScanner.java      |  3 +-
 .../hadoop/hbase/client/ClientSmallScanner.java | 68 ++++++++++++--------
 .../hadoop/hbase/client/ScannerCallable.java    |  4 +-
 .../client/TestClientSmallReversedScanner.java  |  3 +-
 .../hbase/client/TestClientSmallScanner.java    |  3 +-
 .../hadoop/hbase/client/TestFromClientSide.java | 34 ++++++++++
 6 files changed, 84 insertions(+), 31 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/97efb921/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientSmallReversedScanner.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientSmallReversedScanner.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientSmallReversedScanner.java
index 21491ad..c0e3f1f 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientSmallReversedScanner.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientSmallReversedScanner.java
@@ -138,7 +138,8 @@ public class ClientSmallReversedScanner extends ReversedClientScanner
{
     }
 
     smallScanCallable = callableFactory.getCallable(
-        scan, getConnection(), getTable(), localStartKey, cacheNum, this.rpcControllerFactory);
+        scan, getConnection(), getTable(), scanMetrics, localStartKey, cacheNum,
+        rpcControllerFactory);
 
     if (this.scanMetrics != null && regionChanged) {
       this.scanMetrics.countOfRegions.incrementAndGet();

http://git-wip-us.apache.org/repos/asf/hbase/blob/97efb921/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientSmallScanner.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientSmallScanner.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientSmallScanner.java
index 1ff7788..e6c2c8e 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientSmallScanner.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientSmallScanner.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.hbase.client;
 
 import java.io.IOException;
+import java.io.InterruptedIOException;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -29,6 +30,7 @@ import org.apache.hadoop.hbase.KeyValueUtil;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
 import org.apache.hadoop.hbase.ipc.PayloadCarryingRpcController;
 import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
@@ -184,13 +186,48 @@ public class ClientSmallScanner extends ClientScanner {
           + Bytes.toStringBinary(localStartKey) + "'");
     }
     smallScanCallable = callableFactory.getCallable(
-        scan, getConnection(), getTable(), localStartKey, cacheNum, rpcControllerFactory);
+        scan, getConnection(), getTable(), scanMetrics, localStartKey, cacheNum,
+        rpcControllerFactory);
     if (this.scanMetrics != null && regionChanged) {
       this.scanMetrics.countOfRegions.incrementAndGet();
     }
     return true;
   }
 
+  static class SmallScannerCallable extends ScannerCallable {
+    public SmallScannerCallable(HConnection connection, TableName tableName, Scan scan,
+        ScanMetrics scanMetrics, PayloadCarryingRpcController controller, int cacheNum) {
+      super(connection, tableName, scan, scanMetrics, controller);
+      this.setCaching(cacheNum);
+    }
+
+    @Override
+    public Result [] call() throws IOException {
+      if (this.closed) return null;
+      if (Thread.interrupted()) {
+        throw new InterruptedIOException();
+      }
+      ScanRequest request = RequestConverter.buildScanRequest(getLocation()
+        .getRegionInfo().getRegionName(), getScan(), getCaching(), true);
+      ScanResponse response = null;
+      try {
+        controller.setPriority(getTableName());
+        response = getStub().scan(controller, request);
+        Result[] results = ResponseConverter.getResults(controller.cellScanner(), response);
+        if (response.hasMoreResultsInRegion()) {
+          setHasMoreResultsContext(true);
+          setServerHasMoreResults(response.getMoreResultsInRegion());
+        } else {
+          setHasMoreResultsContext(false);
+        }
+        updateResultsMetrics(results);
+        return results;
+      } catch (ServiceException se) {
+        throw ProtobufUtil.getRemoteException(se);
+      }
+    }
+  }
+
   @Override
   public Result next() throws IOException {
     // If the scanner is closed and there's nothing left in the cache, next is a
@@ -276,33 +313,12 @@ public class ClientSmallScanner extends ClientScanner {
   protected static class SmallScannerCallableFactory {
 
     public RegionServerCallable<Result[]> getCallable(final Scan sc, HConnection connection,
-        TableName table, byte[] localStartKey, final int cacheNum,
+        TableName tableName, ScanMetrics scanMetrics, byte[] localStartKey, final int cacheNum,
         final RpcControllerFactory rpcControllerFactory) throws IOException {
       sc.setStartRow(localStartKey);
-      RegionServerCallable<Result[]> callable = new RegionServerCallable<Result[]>(
-          connection, table, sc.getStartRow()) {
-        public Result[] call() throws IOException {
-          ScanRequest request = RequestConverter.buildScanRequest(getLocation()
-            .getRegionInfo().getRegionName(), sc, cacheNum, true);
-          ScanResponse response = null;
-          PayloadCarryingRpcController controller = rpcControllerFactory.newController();
-          try {
-            controller.setPriority(getTableName());
-            response = getStub().scan(controller, request);
-            if (response.hasMoreResultsInRegion()) {
-              setHasMoreResultsContext(true);
-              setServerHasMoreResults(response.getMoreResultsInRegion());
-            } else {
-              setHasMoreResultsContext(false);
-            }
-            return ResponseConverter.getResults(controller.cellScanner(),
-                response);
-          } catch (ServiceException se) {
-            throw ProtobufUtil.getRemoteException(se);
-          }
-        }
-      };
-      return callable;
+      PayloadCarryingRpcController controller = rpcControllerFactory.newController();
+      return new SmallScannerCallable(connection, tableName, sc, scanMetrics, controller,
+        cacheNum);
     }
 
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/97efb921/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java
index cba12a8..f8da289 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java
@@ -64,7 +64,7 @@ public class ScannerCallable extends RegionServerCallable<Result[]>
{
   public static final Log LOG = LogFactory.getLog(ScannerCallable.class);
   private long scannerId = -1L;
   protected boolean instantiated = false;
-  private boolean closed = false;
+  protected boolean closed = false;
   private Scan scan;
   private int caching = 1;
   protected ScanMetrics scanMetrics;
@@ -272,7 +272,7 @@ public class ScannerCallable extends RegionServerCallable<Result[]>
{
     }
   }
 
-  private void updateResultsMetrics(Result[] rrs) {
+  protected void updateResultsMetrics(Result[] rrs) {
     if (this.scanMetrics == null || rrs == null || rrs.length == 0) {
       return;
     }

http://git-wip-us.apache.org/repos/asf/hbase/blob/97efb921/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientSmallReversedScanner.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientSmallReversedScanner.java
b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientSmallReversedScanner.java
index 84932d0..061d6ce 100644
--- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientSmallReversedScanner.java
+++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientSmallReversedScanner.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.KeyValue.Type;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.ClientSmallScanner.SmallScannerCallableFactory;
+import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
 import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
 import org.apache.hadoop.hbase.testclassification.SmallTests;
 import org.junit.After;
@@ -105,7 +106,7 @@ public class TestClientSmallReversedScanner {
     return new SmallScannerCallableFactory() {
       @Override
       public RegionServerCallable<Result[]> getCallable(final Scan sc, HConnection
connection,
-          TableName table, byte[] localStartKey, final int cacheNum,
+          TableName table, ScanMetrics scanMetrics, byte[] localStartKey, final int cacheNum,
           final RpcControllerFactory rpcControllerFactory) {
         return callableWithReplicas;
       }

http://git-wip-us.apache.org/repos/asf/hbase/blob/97efb921/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientSmallScanner.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientSmallScanner.java
b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientSmallScanner.java
index 971020b..debdc59 100644
--- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientSmallScanner.java
+++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientSmallScanner.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.KeyValue.Type;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.ClientSmallScanner.SmallScannerCallableFactory;
+import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
 import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
 import org.apache.hadoop.hbase.testclassification.SmallTests;
 import org.junit.After;
@@ -104,7 +105,7 @@ public class TestClientSmallScanner {
     return new SmallScannerCallableFactory() {
       @Override
       public RegionServerCallable<Result[]> getCallable(final Scan sc, HConnection
connection,
-          TableName table, byte[] localStartKey, final int cacheNum,
+          TableName table, ScanMetrics scanMetrics, byte[] localStartKey, final int cacheNum,
           final RpcControllerFactory rpcControllerFactory) {
         return callableWithReplicas;
       }

http://git-wip-us.apache.org/repos/asf/hbase/blob/97efb921/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
index ddb713f..b543434 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
@@ -59,6 +59,7 @@ import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HRegionLocation;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValueUtil;
 import org.apache.hadoop.hbase.testclassification.LargeTests;
 import org.apache.hadoop.hbase.MiniHBaseCluster;
 import org.apache.hadoop.hbase.ServerName;
@@ -5044,6 +5045,39 @@ public class TestFromClientSide {
     assertEquals("Did not access all the regions in the table", numOfRegions,
         scanMetrics.countOfRegions.get());
 
+    // check byte counters
+    scan = new Scan();
+    scan.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE, Bytes.toBytes(Boolean.TRUE));
+    scan.setCaching(1);
+    scanner = ht.getScanner(scan);
+    int numBytes = 0;
+    for (Result result : scanner.next(1)) {
+      for (Cell cell: result.listCells()) {
+        numBytes += KeyValueUtil.ensureKeyValue(cell).getLength();
+      }
+    }
+    scanner.close();
+    scanMetrics = getScanMetrics(scan);
+    assertEquals("Did not count the result bytes", numBytes,
+      scanMetrics.countOfBytesInResults.get());
+
+    // check byte counters on a small scan
+    scan = new Scan();
+    scan.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE, Bytes.toBytes(Boolean.TRUE));
+    scan.setCaching(1);
+    scan.setSmall(true);
+    scanner = ht.getScanner(scan);
+    numBytes = 0;
+    for (Result result : scanner.next(1)) {
+      for (Cell cell: result.listCells()) {
+        numBytes += KeyValueUtil.ensureKeyValue(cell).getLength();
+      }
+    }
+    scanner.close();
+    scanMetrics = getScanMetrics(scan);
+    assertEquals("Did not count the result bytes", numBytes,
+      scanMetrics.countOfBytesInResults.get());
+
     // now, test that the metrics are still collected even if you don't call close, but do
     // run past the end of all the records
     Scan scanWithoutClose = new Scan();


Mime
View raw message