hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e...@apache.org
Subject [3/5] hbase git commit: HBASE-16678 MapReduce jobs do not update counters from ScanMetrics
Date Fri, 30 Sep 2016 18:31:06 GMT
HBASE-16678 MapReduce jobs do not update counters from ScanMetrics

Conflicts:
	hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java

Conflicts:
	hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9f364084
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9f364084
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9f364084

Branch: refs/heads/branch-1.3
Commit: 9f364084a2a69800b4a4658cb80e3315b881ad8e
Parents: f77f153
Author: Enis Soztutar <enis@apache.org>
Authored: Thu Sep 29 15:34:03 2016 -0700
Committer: Enis Soztutar <enis@apache.org>
Committed: Thu Sep 29 18:56:42 2016 -0700

----------------------------------------------------------------------
 .../hbase/mapreduce/TableRecordReaderImpl.java   |  8 +++++---
 .../hbase/mapreduce/TestTableMapReduce.java      | 19 +++++++++++++++++++
 2 files changed, 24 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/9f364084/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
index 23f59e6..dcc4943 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
@@ -40,6 +40,8 @@ import org.apache.hadoop.mapreduce.InputSplit;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
 import org.apache.hadoop.util.StringUtils;
 
+import com.google.common.annotations.VisibleForTesting;
+
 /**
  * Iterate over an HBase table data, return (ImmutableBytesWritable, Result)
  * pairs.
@@ -53,8 +55,8 @@ public class TableRecordReaderImpl {
   private static final Log LOG = LogFactory.getLog(TableRecordReaderImpl.class);
 
   // HBASE_COUNTER_GROUP_NAME is the name of mapreduce counter group for HBase
-  private static final String HBASE_COUNTER_GROUP_NAME =
-    "HBase Counters";
+  @VisibleForTesting
+  static final String HBASE_COUNTER_GROUP_NAME = "HBase Counters";
   private ResultScanner scanner = null;
   private Scan scan = null;
   private Scan currentScan = null;
@@ -269,7 +271,7 @@ public class TableRecordReaderImpl {
    * @throws IOException
    */
   private void updateCounters() throws IOException {
-    ScanMetrics scanMetrics = this.scan.getScanMetrics();
+    ScanMetrics scanMetrics = currentScan.getScanMetrics();
     if (scanMetrics == null) {
       return;
     }

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f364084/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
index fde1cd9..f5e8b12 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.hbase.mapreduce;
 
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
@@ -38,6 +39,8 @@ import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.Counters;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 import org.junit.experimental.categories.Category;
@@ -114,6 +117,8 @@ public class TestTableMapReduce extends TestTableMapReduceBase {
 
       // verify map-reduce results
       verify(table.getName());
+
+      verifyJobCountersAreEmitted(job);
     } catch (InterruptedException e) {
       throw new IOException(e);
     } catch (ClassNotFoundException e) {
@@ -126,4 +131,18 @@ public class TestTableMapReduce extends TestTableMapReduceBase {
       }
     }
   }
+
+  /**
+   * Verify scan counters are emitted from the job
+   * @param job
+   * @throws IOException
+   */
+  private void verifyJobCountersAreEmitted(Job job) throws IOException {
+    Counters counters = job.getCounters();
+    Counter counter
+      = counters.findCounter(TableRecordReaderImpl.HBASE_COUNTER_GROUP_NAME, "RPC_CALLS");
+    assertNotNull("Unable to find Job counter for HBase scan metrics, RPC_CALLS", counter);
+    assertTrue("Counter value for RPC_CALLS should be larger than 0", counter.getValue()
> 0);
+  }
+
 }


Mime
View raw message