hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nspiegelb...@apache.org
Subject svn commit: r1181965 - in /hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver: HRegion.java StoreScanner.java
Date Tue, 11 Oct 2011 17:44:36 GMT
Author: nspiegelberg
Date: Tue Oct 11 17:44:36 2011
New Revision: 1181965

URL: http://svn.apache.org/viewvc?rev=1181965&view=rev
Log:
Adding a metric for showing the overall size of the result set of get operations

Summary: This metric will basically monitor (sadly, in an ever increasing
fashion) the overall size of the list of KVs passed back as a result of a get
operation.

Test Plan:
Did a load test on the cluster and got appropriate output:

"hadoop.regionserver_cf.actions.getsize": 127235584,
"hadoop.regionserver_cf.info.getsize": 17846,

Currently it will report in bytes, so not sure if we want to compact the
overall reported size to print out in KB or something, but probably not, as
that's a rather useless truncation...

Reviewers: kannan, kranganathan

Reviewed By: kannan

CC: hbase@lists, kannan, bogdan

Differential Revision: 313197

Task ID: 688264

Modified:
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java

Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1181965&r1=1181964&r2=1181965&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java Tue
Oct 11 17:44:36 2011
@@ -273,7 +273,7 @@ public class HRegion implements HeapSize
    *          the ordered set of column families
    * @return a string to print out in metrics
    */
-  private String createMutationSignature(Set<byte[]> families) {
+  public static String createMutationSignature(Set<byte[]> families) {
     int limit = families.size();
     if (1 == limit) {
       return "cf." + Bytes.toString(families.iterator().next());
@@ -308,7 +308,7 @@ public class HRegion implements HeapSize
    *          the family to convert
    * @return the string to print out in metrics
    */
-  private String createMutationSignature(byte[] family) {
+  public static String createMutationSignature(byte[] family) {
     return "cf." + Bytes.toString(family);
   }
 
@@ -1611,7 +1611,7 @@ public class HRegion implements HeapSize
 
     // do after lock
     long after = EnvironmentEdgeManager.currentTimeMillis();
-    String signature = this.createMutationSignature(familyMap.keySet());
+    String signature = HRegion.createMutationSignature(familyMap.keySet());
     HRegion.incrTimeVaryingMetric(signature + ".delete_", after - now);
 
     if (flush) {
@@ -1805,10 +1805,11 @@ public class HRegion implements HeapSize
         // else, if all have been consistent so far, check if it still holds
         // all else, designate failure signature and mark as unclear
         if (null == signature) {
-          signature = this.createMutationSignature(put.getFamilyMap().keySet());
+          signature = HRegion.createMutationSignature(put.getFamilyMap()
+              .keySet());
         } else {
           if (isSignatureClear) {
-            if (!signature.equals(this.createMutationSignature(put
+            if (!signature.equals(HRegion.createMutationSignature(put
                 .getFamilyMap().keySet()))) {
               isSignatureClear = false;
               signature = "cf.__unknown";
@@ -2094,7 +2095,7 @@ public class HRegion implements HeapSize
 
     // do after lock
     long after = EnvironmentEdgeManager.currentTimeMillis();
-    String signature = this.createMutationSignature(familyMap.keySet());
+    String signature = HRegion.createMutationSignature(familyMap.keySet());
     HRegion.incrTimeVaryingMetric(signature + ".put_", after - now);
 
     if (flush) {
@@ -3370,7 +3371,7 @@ public class HRegion implements HeapSize
 
     // do after lock
     long after = EnvironmentEdgeManager.currentTimeMillis();
-    String signature = this.createMutationSignature(get.familySet());
+    String signature = HRegion.createMutationSignature(get.familySet());
     HRegion.incrTimeVaryingMetric(signature + ".get_", after - now);
 
     return results;
@@ -3443,7 +3444,7 @@ public class HRegion implements HeapSize
 
     // do after lock
     long after = EnvironmentEdgeManager.currentTimeMillis();
-    String signature = this.createMutationSignature(family);
+    String signature = HRegion.createMutationSignature(family);
     HRegion.incrTimeVaryingMetric(signature + ".increment_", after - before);
 
     if (flush) {

Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java?rev=1181965&r1=1181964&r2=1181965&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
(original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
Tue Oct 11 17:44:36 2011
@@ -20,16 +20,17 @@
 
 package org.apache.hadoop.hbase.regionserver;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NavigableSet;
+
 import org.apache.commons.lang.NotImplementedException;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.client.Scan;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.NavigableSet;
+import org.apache.hadoop.hbase.util.Bytes;
 
 /**
  * Scanner scans both the memstore and the HStore. Coalesce KeyValue stream
@@ -43,7 +44,7 @@ class StoreScanner implements KeyValueSc
   private boolean cacheBlocks;
   private int countPerRow = 0;
   private int storeLimit;
-
+  private String metricNameGetsize;
   // Used to indicate that the scanner has closed (see HBASE-1107)
   // Doesnt need to be volatile because it's always accessed via synchronized methods
   private boolean closing = false;
@@ -63,6 +64,8 @@ class StoreScanner implements KeyValueSc
   StoreScanner(Store store, Scan scan, final NavigableSet<byte[]> columns)
                               throws IOException {
     this.store = store;
+    initializeMetricNames();
+
     this.cacheBlocks = scan.getCacheBlocks();
     matcher = new ScanQueryMatcher(scan, store.getFamily().getName(),
         columns, store.ttl, store.comparator.getRawComparator(),
@@ -104,6 +107,8 @@ class StoreScanner implements KeyValueSc
                             boolean retainDeletesInOutput)
       throws IOException {
     this.store = store;
+    this.initializeMetricNames();
+
     this.cacheBlocks = false;
     this.isGet = false;
     matcher = new ScanQueryMatcher(scan, store.getFamily().getName(),
@@ -127,6 +132,8 @@ class StoreScanner implements KeyValueSc
       final List<KeyValueScanner> scanners)
         throws IOException {
     this.store = null;
+    this.initializeMetricNames();
+
     this.isGet = false;
     this.cacheBlocks = scan.getCacheBlocks();
     this.matcher = new ScanQueryMatcher(scan, colFamily, columns, ttl,
@@ -140,6 +147,23 @@ class StoreScanner implements KeyValueSc
     heap = new KeyValueHeap(scanners, comparator);
   }
 
+  /**
+   * Method used internally to initialize metric names throughout the
+   * constructors.
+   *
+   * To be called after the store variable has been initialized!
+   */
+  private void initializeMetricNames() {
+    byte[] family;
+    if (null != this.store) {
+      family = this.store.getFamily().getName();
+    } else {
+      family = Bytes.toBytes("__unknown");
+    }
+    String mutationSignature = HRegion.createMutationSignature(family);
+    this.metricNameGetsize = mutationSignature + ".getsize";
+  }
+
   /*
    * @return List of scanners ordered properly.
    */
@@ -186,6 +210,7 @@ class StoreScanner implements KeyValueSc
     return scanners;
   }
 
+  @Override
   public synchronized KeyValue peek() {
     if (this.heap == null) {
       return this.lastTop;
@@ -193,11 +218,13 @@ class StoreScanner implements KeyValueSc
     return this.heap.peek();
   }
 
+  @Override
   public KeyValue next() {
     // throw runtime exception perhaps?
     throw new RuntimeException("Never call StoreScanner.next()");
   }
 
+  @Override
   public synchronized void close() {
     if (this.closing) return;
     this.closing = true;
@@ -210,6 +237,7 @@ class StoreScanner implements KeyValueSc
     this.lastTop = null; // If both are null, we are closed.
   }
 
+  @Override
   public synchronized boolean seek(KeyValue key) throws IOException {
     if (this.heap == null) {
 
@@ -227,6 +255,7 @@ class StoreScanner implements KeyValueSc
    * @param limit
    * @return true if there are more rows, false if scanner is done
    */
+  @Override
   public synchronized boolean next(List<KeyValue> outResult, int limit) throws IOException
{
     //DebugPrint.println("SS.next");
 
@@ -269,6 +298,7 @@ class StoreScanner implements KeyValueSc
             break LOOP;
           }
 
+          HRegion.incrNumericMetric(this.metricNameGetsize, copyKv.getLength());
           results.add(copyKv);
           this.heap.next();
           if (limit > 0 && (results.size() == limit)) {
@@ -333,11 +363,13 @@ class StoreScanner implements KeyValueSc
     return false;
   }
 
+  @Override
   public synchronized boolean next(List<KeyValue> outResult) throws IOException {
     return next(outResult, -1);
   }
 
   // Implementation of ChangedReadersObserver
+  @Override
   public synchronized void updateReaders() throws IOException {
     if (this.closing) return;
 



Mime
View raw message