hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nspiegelb...@apache.org
Subject svn commit: r1210640 - in /hbase/trunk/src: main/java/org/apache/hadoop/hbase/io/hfile/ main/java/org/apache/hadoop/hbase/regionserver/ main/java/org/apache/hadoop/hbase/regionserver/metrics/ test/java/org/apache/hadoop/hbase/io/hfile/
Date Mon, 05 Dec 2011 21:17:48 GMT
Author: nspiegelberg
Date: Mon Dec  5 21:17:47 2011
New Revision: 1210640

URL: http://svn.apache.org/viewvc?rev=1210640&view=rev
Log:
Ability to calculate the blockcache hit ratio for the last few minutes

Summary:
The metric blockcacheHitRatio is since the beginning of time. It would be nice
to calculate the block cache hit ratio for the past few minutes.

This patch remembers the blockcacheHitRatio for the last 5 periods by default.

Test Plan: unit test attached.

Reviewers: jgray, khemani, nspiegelberg, JIRA

Reviewed By: nspiegelberg

CC: nspiegelberg, dhruba

Differential Revision: 585

Modified:
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java?rev=1210640&r1=1210639&r2=1210640&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java Mon Dec  5
21:17:47 2011
@@ -25,6 +25,12 @@ import java.util.concurrent.atomic.Atomi
  * Class that implements cache metrics.
  */
 public class CacheStats {
+
+  /** Sliding window statistics. The number of metric periods to include in
+   * sliding window hit ratio calculations.
+   */
+  static final int DEFAULT_WINDOW_PERIODS = 5;
+
   /** The number of getBlock requests that were cache hits */
   private final AtomicLong hitCount = new AtomicLong(0);
   /**
@@ -46,6 +52,39 @@ public class CacheStats {
   /** The total number of blocks that have been evicted */
   private final AtomicLong evictedBlockCount = new AtomicLong(0);
 
+  /** The number of metrics periods to include in window */
+  private final int numPeriodsInWindow;
+  /** Hit counts for each period in window */
+  private final long [] hitCounts;
+  /** Caching hit counts for each period in window */
+  private final long [] hitCachingCounts;
+  /** Access counts for each period in window */
+  private final long [] requestCounts;
+  /** Caching access counts for each period in window */
+  private final long [] requestCachingCounts;
+  /** Last hit count read */
+  private long lastHitCount = 0;
+  /** Last hit caching count read */
+  private long lastHitCachingCount = 0;
+  /** Last request count read */
+  private long lastRequestCount = 0;
+  /** Last request caching count read */
+  private long lastRequestCachingCount = 0;
+  /** Current window index (next to be updated) */
+  private int windowIndex = 0;
+
+  public CacheStats() {
+    this(DEFAULT_WINDOW_PERIODS);
+  }
+
+  public CacheStats(int numPeriodsInWindow) {
+    this.numPeriodsInWindow = numPeriodsInWindow;
+    this.hitCounts = initializeZeros(numPeriodsInWindow);
+    this.hitCachingCounts = initializeZeros(numPeriodsInWindow);
+    this.requestCounts = initializeZeros(numPeriodsInWindow);
+    this.requestCachingCounts = initializeZeros(numPeriodsInWindow);
+  }
+
   public void miss(boolean caching) {
     missCount.incrementAndGet();
     if (caching) missCachingCount.incrementAndGet();
@@ -115,4 +154,43 @@ public class CacheStats {
   public double evictedPerEviction() {
     return ((float)getEvictedCount()/(float)getEvictionCount());
   }
-}
\ No newline at end of file
+
+  public void rollMetricsPeriod() {
+    hitCounts[windowIndex] = getHitCount() - lastHitCount;
+    lastHitCount = getHitCount();
+    hitCachingCounts[windowIndex] =
+      getHitCachingCount() - lastHitCachingCount;
+    lastHitCachingCount = getHitCachingCount();
+    requestCounts[windowIndex] = getRequestCount() - lastRequestCount;
+    lastRequestCount = getRequestCount();
+    requestCachingCounts[windowIndex] =
+      getRequestCachingCount() - lastRequestCachingCount;
+    lastRequestCachingCount = getRequestCachingCount();
+    windowIndex = (windowIndex + 1) % numPeriodsInWindow;
+  }
+
+  public double getHitRatioPastNPeriods() {
+    double ratio = ((double)sum(hitCounts)/(double)sum(requestCounts));
+    return Double.isNaN(ratio) ? 0 : ratio;
+  }
+
+  public double getHitCachingRatioPastNPeriods() {
+    double ratio =
+      ((double)sum(hitCachingCounts)/(double)sum(requestCachingCounts));
+    return Double.isNaN(ratio) ? 0 : ratio;
+  }
+
+  private static long sum(long [] counts) {
+    long sum = 0;
+    for (long count : counts) sum += count;
+    return sum;
+  }
+
+  private static long [] initializeZeros(int n) {
+    long [] zeros = new long [n];
+    for (int i=0; i<n; i++) {
+      zeros[i] = 0L;
+    }
+    return zeros;
+  }
+}

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=1210640&r1=1210639&r2=1210640&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Mon
Dec  5 21:17:47 2011
@@ -1405,6 +1405,14 @@ public class HRegionServer implements HR
       ratio = blockCache.getStats().getHitCachingRatio();
       percent = (int) (ratio * 100);
       this.metrics.blockCacheHitCachingRatio.set(percent);
+      // past N period block cache hit / hit caching ratios
+      cacheStats.rollMetricsPeriod();
+      ratio = cacheStats.getHitRatioPastNPeriods();
+      percent = (int) (ratio * 100);
+      this.metrics.blockCacheHitRatioPastNPeriods.set(percent);
+      ratio = cacheStats.getHitCachingRatioPastNPeriods();
+      percent = (int) (ratio * 100);
+      this.metrics.blockCacheHitCachingRatioPastNPeriods.set(percent);
     }
     float localityIndex = hdfsBlocksDistribution.getBlockLocalityIndex(
       getServerName().getHostname());

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java?rev=1210640&r1=1210639&r2=1210640&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
(original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
Mon Dec  5 21:17:47 2011
@@ -113,6 +113,12 @@ public class RegionServerMetrics impleme
    */
   public final MetricsIntValue blockCacheHitCachingRatio = new MetricsIntValue("blockCacheHitCachingRatio",
registry);
 
+  /** Block hit ratio for past N periods. */
+  public final MetricsIntValue blockCacheHitRatioPastNPeriods = new MetricsIntValue("blockCacheHitRatioPastNPeriods",
registry);
+
+  /** Block hit caching ratio for past N periods */
+  public final MetricsIntValue blockCacheHitCachingRatioPastNPeriods = new MetricsIntValue("blockCacheHitCachingRatioPastNPeriods",
registry);
+
   /*
    * Count of requests to the regionservers since last call to metrics update
    */
@@ -295,6 +301,8 @@ public class RegionServerMetrics impleme
       this.blockCacheHitRatio.pushMetric(this.metricsRecord);
       this.blockCacheHitCachingRatio.pushMetric(this.metricsRecord);
       this.hdfsBlocksLocalityIndex.pushMetric(this.metricsRecord);
+      this.blockCacheHitRatioPastNPeriods.pushMetric(this.metricsRecord);
+      this.blockCacheHitCachingRatioPastNPeriods.pushMetric(this.metricsRecord);
 
       // Mix in HFile and HLog metrics
       // Be careful. Here is code for MTVR from up in hadoop:

Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java?rev=1210640&r1=1210639&r2=1210640&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java Mon
Dec  5 21:17:47 2011
@@ -506,6 +506,96 @@ public class TestLruBlockCache {
     }
   }
 
+  // test metricsPastNPeriods
+  @Test
+  public void testPastNPeriodsMetrics() throws Exception {
+   double delta = 0.01;
+
+    // 3 total periods
+    CacheStats stats = new CacheStats(3);
+
+    // No accesses, should be 0
+    stats.rollMetricsPeriod();
+    assertEquals(0.0, stats.getHitRatioPastNPeriods(), delta);
+    assertEquals(0.0, stats.getHitCachingRatioPastNPeriods(), delta);
+
+    // period 1, 1 hit caching, 1 hit non-caching, 2 miss non-caching
+    // should be (2/4)=0.5 and (1/1)=1
+    stats.hit(false);
+    stats.hit(true);
+    stats.miss(false);
+    stats.miss(false);
+    stats.rollMetricsPeriod();
+    assertEquals(0.5, stats.getHitRatioPastNPeriods(), delta);
+    assertEquals(1.0, stats.getHitCachingRatioPastNPeriods(), delta);
+
+    // period 2, 1 miss caching, 3 miss non-caching
+    // should be (2/8)=0.25 and (1/2)=0.5
+    stats.miss(true);
+    stats.miss(false);
+    stats.miss(false);
+    stats.miss(false);
+    stats.rollMetricsPeriod();
+    assertEquals(0.25, stats.getHitRatioPastNPeriods(), delta);
+    assertEquals(0.5, stats.getHitCachingRatioPastNPeriods(), delta);
+
+    // period 3, 2 hits of each type
+    // should be (6/12)=0.5 and (3/4)=0.75
+    stats.hit(false);
+    stats.hit(true);
+    stats.hit(false);
+    stats.hit(true);
+    stats.rollMetricsPeriod();
+    assertEquals(0.5, stats.getHitRatioPastNPeriods(), delta);
+    assertEquals(0.75, stats.getHitCachingRatioPastNPeriods(), delta);
+
+    // period 4, evict period 1, two caching misses
+    // should be (4/10)=0.4 and (2/5)=0.4
+    stats.miss(true);
+    stats.miss(true);
+    stats.rollMetricsPeriod();
+    assertEquals(0.4, stats.getHitRatioPastNPeriods(), delta);
+    assertEquals(0.4, stats.getHitCachingRatioPastNPeriods(), delta);
+
+    // period 5, evict period 2, 2 caching misses, 2 non-caching hit
+    // should be (6/10)=0.6 and (2/6)=1/3
+    stats.miss(true);
+    stats.miss(true);
+    stats.hit(false);
+    stats.hit(false);
+    stats.rollMetricsPeriod();
+    assertEquals(0.6, stats.getHitRatioPastNPeriods(), delta);
+    assertEquals((double)1/3, stats.getHitCachingRatioPastNPeriods(), delta);
+
+    // period 6, evict period 3
+    // should be (2/6)=1/3 and (0/4)=0
+    stats.rollMetricsPeriod();
+    assertEquals((double)1/3, stats.getHitRatioPastNPeriods(), delta);
+    assertEquals(0.0, stats.getHitCachingRatioPastNPeriods(), delta);
+
+    // period 7, evict period 4
+    // should be (2/4)=0.5 and (0/2)=0
+    stats.rollMetricsPeriod();
+    assertEquals(0.5, stats.getHitRatioPastNPeriods(), delta);
+    assertEquals(0.0, stats.getHitCachingRatioPastNPeriods(), delta);
+
+    // period 8, evict period 5
+    // should be 0 and 0
+    stats.rollMetricsPeriod();
+    assertEquals(0.0, stats.getHitRatioPastNPeriods(), delta);
+    assertEquals(0.0, stats.getHitCachingRatioPastNPeriods(), delta);
+
+    // period 9, one of each
+    // should be (2/4)=0.5 and (1/2)=0.5
+    stats.miss(true);
+    stats.miss(false);
+    stats.hit(true);
+    stats.hit(false);
+    stats.rollMetricsPeriod();
+    assertEquals(0.5, stats.getHitRatioPastNPeriods(), delta);
+    assertEquals(0.5, stats.getHitCachingRatioPastNPeriods(), delta);
+  }
+
   private CachedItem [] generateFixedBlocks(int numBlocks, int size, String pfx) {
     CachedItem [] blocks = new CachedItem[numBlocks];
     for(int i=0;i<numBlocks;i++) {



Mime
View raw message