hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nspiegelb...@apache.org
Subject svn commit: r1181433 - in /hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase: io/hfile/HFile.java regionserver/metrics/RegionServerMetrics.java regionserver/wal/HLog.java
Date Tue, 11 Oct 2011 02:08:04 GMT
Author: nspiegelberg
Date: Tue Oct 11 02:08:04 2011
New Revision: 1181433

URL: http://svn.apache.org/viewvc?rev=1181433&view=rev
Log:
Add writeSize metric

Summary:
Add writeSize variable to JMX so we can track average write sizes and see how
closely they correlate with put latency

Test Plan:
tested on cluster

DiffCamp Revision: 176576
Reviewed By: kannan
CC: kannan
Revert Plan:
OK

Modified:
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java

Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java?rev=1181433&r1=1181432&r2=1181433&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java Tue Oct
11 02:08:04 2011
@@ -170,13 +170,13 @@ public class HFile {
     DEFAULT_COMPRESSION_ALGORITHM.getName();
 
   // For measuring latency of "typical" reads and writes
-  private static volatile long readOps;
+  private static volatile int readOps;
   private static volatile long readTime;
-  private static volatile long writeOps;
+  private static volatile int writeOps;
   private static volatile long writeTime;
 
-  public static final long getReadOps() {
-    long ret = readOps;
+  public static final int getReadOps() {
+    int ret = readOps;
     readOps = 0;
     return ret;
   }
@@ -187,8 +187,8 @@ public class HFile {
     return ret;
   }
 
-  public static final long getWriteOps() {
-    long ret = writeOps;
+  public static final int getWriteOps() {
+    int ret = writeOps;
     writeOps = 0;
     return ret;
   }

Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java?rev=1181433&r1=1181432&r2=1181433&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
(original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
Tue Oct 11 02:08:04 2011
@@ -137,6 +137,12 @@ public class RegionServerMetrics impleme
     new MetricsTimeVaryingRate("fsWriteLatency", registry);
 
   /**
+   * size (in bytes) of data in HLog append calls
+   */
+  public final MetricsTimeVaryingRate fsWriteSize =
+    new MetricsTimeVaryingRate("fsWriteSize", registry);
+
+  /**
    * filesystem sync latency
    */
   public final MetricsTimeVaryingRate fsSyncLatency =
@@ -222,7 +228,6 @@ public class RegionServerMetrics impleme
       this.blockCacheCount.pushMetric(this.metricsRecord);
       this.blockCacheHitRatio.pushMetric(this.metricsRecord);
 
-      // Mix in HFile and HLog metrics
       // Be careful. Here is code for MTVR from up in hadoop:
       // public synchronized void inc(final int numOps, final long time) {
       //   currentData.numOperations += numOps;
@@ -231,15 +236,23 @@ public class RegionServerMetrics impleme
       //    minMax.update(timePerOps);
       // }
       // Means you can't pass a numOps of zero or get a ArithmeticException / by zero.
-      int ops = (int)HFile.getReadOps();
-      if (ops != 0) this.fsReadLatency.inc(ops, HFile.getReadTime());
-      ops = (int)HFile.getWriteOps();
-      if (ops != 0) this.fsWriteLatency.inc(ops, HFile.getWriteTime());
-      // mix in HLog metrics
-      ops = (int)HLog.getWriteOps();
-      if (ops != 0) this.fsWriteLatency.inc(ops, HLog.getWriteTime());
-      ops = (int)HLog.getSyncOps();
+      // HLog metrics
+      int ops = HLog.getWriteOps();
+      if (ops != 0) {
+        this.fsWriteLatency.inc(ops, HLog.getWriteTime());
+        this.fsWriteSize.inc(ops, HLog.getWriteSize());
+      }
+      ops = HLog.getSyncOps();
       if (ops != 0) this.fsSyncLatency.inc(ops, HLog.getSyncTime());
+      // HFile metrics
+      ops = HFile.getReadOps();
+      if (ops != 0) this.fsReadLatency.inc(ops, HFile.getReadTime());
+      /* NOTE: removed HFile write latency.  2 reasons:
+       * 1) Mixing HLog latencies are far higher priority since they're
+       *      on-demand and HFile is used in background (compact/flush)
+       * 2) HFile metrics are being handled at a higher level
+       *      by compaction & flush metrics.
+       */
 
       // push the result
       this.fsReadLatency.pushMetric(this.metricsRecord);

Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java?rev=1181433&r1=1181432&r2=1181433&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java Tue
Oct 11 02:08:04 2011
@@ -249,17 +249,18 @@ public class HLog implements Syncable {
   }
 
   // For measuring latency of writes
-  private static volatile long writeOps;
+  private static volatile int writeOps;
   private static volatile long writeTime;
+  private static volatile long writeSize;
   // For measuring latency of syncs
-  private static volatile long syncOps;
+  private static volatile int syncOps;
   private static volatile long syncTime;
 
   public static volatile long lastSplitTime = 0;
   public static volatile long lastSplitSize = 0;
 
-  public static long getWriteOps() {
-    long ret = writeOps;
+  public static int getWriteOps() {
+    int ret = writeOps;
     writeOps = 0;
     return ret;
   }
@@ -270,8 +271,14 @@ public class HLog implements Syncable {
     return ret;
   }
 
-  public static long getSyncOps() {
-    long ret = syncOps;
+  public static long getWriteSize() {
+    long ret = writeSize;
+    writeSize = 0;
+    return ret;
+  }
+
+  public static int getSyncOps() {
+    int ret = syncOps;
     syncOps = 0;
     return ret;
   }
@@ -1056,11 +1063,12 @@ public class HLog implements Syncable {
       long took = System.currentTimeMillis() - now;
       writeTime += took;
       writeOps++;
+      long len = 0;
+      for(KeyValue kv : logEdit.getKeyValues()) {
+        len += kv.getLength();
+      }
+      writeSize += len;
       if (took > 1000) {
-        long len = 0;
-        for(KeyValue kv : logEdit.getKeyValues()) {
-          len += kv.getLength();
-        }
         LOG.warn(String.format(
           "%s took %d ms appending an edit to hlog; editcount=%d, len~=%s",
           Thread.currentThread().getName(), took, this.numEntries.get(),
@@ -1134,6 +1142,11 @@ public class HLog implements Syncable {
         this.writer.append(new Entry(key, edit));
         writeTime += System.currentTimeMillis() - now;
         writeOps++;
+        long len = 0;
+        for(KeyValue kv : edit.getKeyValues()) {
+          len += kv.getLength();
+        }
+        writeSize += len;
         this.numEntries.incrementAndGet();
         Long seq = this.lastSeqWritten.get(regionName);
         if (seq != null && logSeqId >= seq.longValue()) {



Mime
View raw message