hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From la...@apache.org
Subject svn commit: r1404464 - in /hbase/branches/0.94/src: main/java/org/apache/hadoop/hbase/io/hfile/ main/java/org/apache/hadoop/hbase/regionserver/metrics/ test/java/org/apache/hadoop/hbase/regionserver/metrics/
Date Thu, 01 Nov 2012 04:58:54 GMT
Author: larsh
Date: Thu Nov  1 04:58:53 2012
New Revision: 1404464

URL: http://svn.apache.org/viewvc?rev=1404464&view=rev
Log:
HBASE-6852 SchemaMetrics.updateOnCacheHit costs too much while full scanning a table with
all of its fields (Cheng Hao and LarsH) - REAPPLY

Modified:
    hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
    hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
    hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java
    hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestSchemaMetrics.java

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java?rev=1404464&r1=1404463&r2=1404464&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
(original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
Thu Nov  1 04:58:53 2012
@@ -391,6 +391,8 @@ public class HFileReaderV1 extends Abstr
       this.istream.close();
       this.istream = null;
     }
+    
+    getSchemaMetrics().flushMetrics();
   }
 
   protected abstract static class AbstractScannerV1

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java?rev=1404464&r1=1404463&r2=1404464&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
(original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
Thu Nov  1 04:58:53 2012
@@ -424,6 +424,8 @@ public class HFileReaderV2 extends Abstr
         istream = null;
       }
     }
+    
+    getSchemaMetrics().flushMetrics();
   }
 
   protected abstract static class AbstractScannerV2

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java?rev=1404464&r1=1404463&r2=1404464&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java
(original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java
Thu Nov  1 04:58:53 2012
@@ -29,6 +29,7 @@ import java.util.Set;
 import java.util.TreeMap;
 import java.util.TreeSet;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLongArray;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -38,7 +39,6 @@ import org.apache.commons.logging.LogFac
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
-import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
 
@@ -236,6 +236,9 @@ public class SchemaMetrics {
   public static final SchemaMetrics ALL_SCHEMA_METRICS =
     getInstance(TOTAL_KEY, TOTAL_KEY);
 
+  /** Threshold for flush the metrics, currently used only for "on cache hit" */
+  private static final long THRESHOLD_METRICS_FLUSH = 100l;
+
   /**
    * Whether to include table name in metric names. If this is null, it has not
    * been initialized. This is a global instance, but we also have a copy of it
@@ -254,6 +257,8 @@ public class SchemaMetrics {
   private final String[] bloomMetricNames = new String[2];
   private final String[] storeMetricNames = new String[NUM_STORE_METRIC_TYPES];
   private final String[] storeMetricNamesMax = new String[NUM_STORE_METRIC_TYPES];
+  private final AtomicLongArray onHitCacheMetrics= 
+      new AtomicLongArray(NUM_BLOCK_CATEGORIES * BOOL_VALUES.length);
 
   private SchemaMetrics(final String tableName, final String cfName) {
     String metricPrefix = SchemaMetrics.generateSchemaMetricsPrefix(
@@ -261,6 +266,9 @@ public class SchemaMetrics {
 
     for (BlockCategory blockCategory : BlockCategory.values()) {
       for (boolean isCompaction : BOOL_VALUES) {
+        // initialize the cache metrics
+        onHitCacheMetrics.set(getCacheHitMetricIndex(blockCategory, isCompaction), 0);
+        
         for (BlockMetricType metricType : BlockMetricType.values()) {
           if (!metricType.compactionAware && isCompaction) {
             continue;
@@ -336,6 +344,11 @@ public class SchemaMetrics {
     return existingMetrics != null ? existingMetrics : schemaMetrics;
   }
 
+  private static final int getCacheHitMetricIndex (BlockCategory blockCategory,
+      boolean isCompaction) {
+    return blockCategory.ordinal() * BOOL_VALUES.length + (isCompaction ? 1 : 0);
+  }
+  
   private static final int getBlockMetricIndex(BlockCategory blockCategory,
       boolean isCompaction, BlockMetricType metricType) {
     int i = 0;
@@ -365,15 +378,24 @@ public class SchemaMetrics {
    */
   private void incrNumericMetric(BlockCategory blockCategory,
       boolean isCompaction, BlockMetricType metricType) {
+    incrNumericMetric (blockCategory, isCompaction, metricType, 1);
+  }
+  
+  /**
+   * Increments the given metric, both per-CF and aggregate, for both the given
+   * category and all categories in aggregate (four counters total).
+   */
+  private void incrNumericMetric(BlockCategory blockCategory,
+      boolean isCompaction, BlockMetricType metricType, long amount) {
     if (blockCategory == null) {
       blockCategory = BlockCategory.UNKNOWN;  // So that we see this in stats.
     }
     RegionMetricsStorage.incrNumericMetric(getBlockMetricName(blockCategory,
-        isCompaction, metricType), 1);
+        isCompaction, metricType), amount);
 
     if (blockCategory != BlockCategory.ALL_CATEGORIES) {
       incrNumericMetric(BlockCategory.ALL_CATEGORIES, isCompaction,
-          metricType);
+          metricType, amount);
     }
   }
 
@@ -445,15 +467,60 @@ public class SchemaMetrics {
    */
   public void updateOnCacheHit(BlockCategory blockCategory,
       boolean isCompaction) {
+    updateOnCacheHit(blockCategory, isCompaction, 1);
+  }
+  
+  /**
+   * Updates the number of hits and the total number of block reads on a block
+   * cache hit.
+   */
+  public void updateOnCacheHit(BlockCategory blockCategory,
+      boolean isCompaction, long count) {
     blockCategory.expectSpecific();
-    incrNumericMetric(blockCategory, isCompaction, BlockMetricType.CACHE_HIT);
-    incrNumericMetric(blockCategory, isCompaction, BlockMetricType.READ_COUNT);
-    if (this != ALL_SCHEMA_METRICS) {
-      ALL_SCHEMA_METRICS.updateOnCacheHit(blockCategory, isCompaction);
+    int idx = getCacheHitMetricIndex(blockCategory, isCompaction);
+    
+    if (this.onHitCacheMetrics.addAndGet(idx, count) > THRESHOLD_METRICS_FLUSH) {
+      flushCertainOnCacheHitMetrics(blockCategory, isCompaction);
+    }
+  }
+  
+  private void flushCertainOnCacheHitMetrics(BlockCategory blockCategory, boolean isCompaction)
{
+    int idx = getCacheHitMetricIndex(blockCategory, isCompaction);
+    long tempCount = this.onHitCacheMetrics.getAndSet(idx, 0);
+    
+    if (tempCount > 0) {
+      incrNumericMetric(blockCategory, isCompaction, BlockMetricType.CACHE_HIT, tempCount);
+      incrNumericMetric(blockCategory, isCompaction, BlockMetricType.READ_COUNT, tempCount);
+      if (this != ALL_SCHEMA_METRICS) {
+        ALL_SCHEMA_METRICS.updateOnCacheHit(blockCategory, isCompaction, tempCount);
+      }
+    }
+  }
+  
+  /**
+   * Flush the on cache hit metrics;
+   */
+  private void flushOnCacheHitMetrics() {
+    for (BlockCategory blockCategory : BlockCategory.values()) {
+      for (boolean isCompaction : BOOL_VALUES) {
+        flushCertainOnCacheHitMetrics (blockCategory, isCompaction);
+      }
     }
   }
 
   /**
+   * Notify the SchemaMetrics to flush all of the the metrics
+   */
+  public void flushMetrics() {
+    // currently only for "on cache hit metrics"
+    flushOnCacheHitMetrics();
+    
+    if (this != ALL_SCHEMA_METRICS) {
+      ALL_SCHEMA_METRICS.flushMetrics();
+    }
+  }
+  
+  /**
    * Updates read time, the number of misses, and the total number of block
    * reads on a block cache miss.
    */
@@ -746,6 +813,7 @@ public class SchemaMetrics {
   public static Map<String, Long> getMetricsSnapshot() {
     Map<String, Long> metricsSnapshot = new TreeMap<String, Long>();
     for (SchemaMetrics cfm : tableAndFamilyToMetrics.values()) {
+      cfm.flushMetrics();
       for (String metricName : cfm.getAllMetricNames()) {
         long metricValue;
         if (isTimeVaryingKey(metricName)) {
@@ -796,6 +864,7 @@ public class SchemaMetrics {
   }
 
   public static void validateMetricChanges(Map<String, Long> oldMetrics) {
+    
     final Map<String, Long> newMetrics = getMetricsSnapshot();
     final Map<String, Long> allCfDeltas = new TreeMap<String, Long>();
     final Map<String, Long> allBlockCategoryDeltas =

Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestSchemaMetrics.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestSchemaMetrics.java?rev=1404464&r1=1404463&r2=1404464&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestSchemaMetrics.java
(original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestSchemaMetrics.java
Thu Nov  1 04:58:53 2012
@@ -201,8 +201,10 @@ public class TestSchemaMetrics {
 
           for (boolean isCompaction : BOOL_VALUES) {
             sm.updateOnCacheHit(blockCat, isCompaction);
+            sm.flushMetrics();
             checkMetrics();
             sm.updateOnCacheMiss(blockCat, isCompaction, rand.nextInt());
+            sm.flushMetrics();
             checkMetrics();
           }
 



Mime
View raw message