hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mbau...@apache.org
Subject svn commit: r1293043 - in /hbase/branches/0.89-fb/src/main: java/org/apache/hadoop/hbase/ java/org/apache/hadoop/hbase/metrics/ java/org/apache/hadoop/hbase/regionserver/ resources/hbase-webapps/regionserver/
Date Fri, 24 Feb 2012 00:50:31 GMT
Author: mbautin
Date: Fri Feb 24 00:50:31 2012
New Revision: 1293043

URL: http://svn.apache.org/viewvc?rev=1293043&view=rev
Log:
[jira][HBASE-5407][89-fb] Show the per-region level request/sec count in the web  ui

Summary:
It would be nice to show the per-region level request/sec count in the
web ui, especially when debugging the hot region problem.
Open source revision: https://reviews.facebook.net/D1779

Test Plan: Tested on the dev cluster

Reviewers: Kannan, Karthik, khemani

Reviewed By: khemani
CC: JIRA

Added:
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/metrics/RequestMetrics.java
Modified:
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HServerLoad.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
    hbase/branches/0.89-fb/src/main/resources/hbase-webapps/regionserver/regionserver.jsp

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HServerLoad.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HServerLoad.java?rev=1293043&r1=1293042&r2=1293043&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HServerLoad.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HServerLoad.java Fri Feb
24 00:50:31 2012
@@ -26,6 +26,7 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 
+import org.apache.hadoop.hbase.metrics.RequestMetrics;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Strings;
 import org.apache.hadoop.io.Writable;
@@ -89,6 +90,15 @@ public class HServerLoad implements Writ
      */
     private int totalStaticBloomSizeKB = -1;
 
+    /** The total read request count */
+    private long totalReadRequest = 0;
+    /** The total write request count */
+    private long totalWriteRequest = 0;
+    /** The total read request per sec*/
+    private int readRequestPerSec = 0;
+    /** The total write request per sec*/
+    private int writeRequestPerSec = 0;
+
     /**
      * Constructor, for Writable
      */
@@ -109,6 +119,29 @@ public class HServerLoad implements Writ
         final int memstoreSizeMB, final int storefileIndexSizeMB,
         final int rootIndexSizeKB, final int totalStaticIndexSizeKB,
         final int totalStaticBloomSizeKB) {
+      this(name, stores, storefiles, storefileSizeMB, memstoreSizeMB,
+          storefileIndexSizeMB, rootIndexSizeKB, totalStaticIndexSizeKB,
+          totalStaticBloomSizeKB, null, null);
+    }
+
+    /**
+     * @param name
+     * @param stores
+     * @param storefiles
+     * @param storefileSizeMB
+     * @param memstoreSizeMB
+     * @param storefileIndexSizeMB
+     * @param totalReadRequest
+     * @param totalReadRequest
+     * @param readRequestPerSec
+     * @param readRequestPerSec
+     */
+    public RegionLoad(final byte[] name, final int stores,
+        final int storefiles, final int storefileSizeMB,
+        final int memstoreSizeMB, final int storefileIndexSizeMB,
+        final int rootIndexSizeKB, final int totalStaticIndexSizeKB,
+        final int totalStaticBloomSizeKB, RequestMetrics readRequest,
+        RequestMetrics writeRequest) {
       this.name = name;
       this.stores = stores;
       this.storefiles = storefiles;
@@ -118,6 +151,14 @@ public class HServerLoad implements Writ
       this.rootIndexSizeKB = rootIndexSizeKB;
       this.totalStaticIndexSizeKB = totalStaticIndexSizeKB;
       this.totalStaticBloomSizeKB = totalStaticBloomSizeKB;
+      if (readRequest != null) {
+        this.totalReadRequest = readRequest.getTotalRequestCount();
+        this.readRequestPerSec = readRequest.getRequestPerSecond();
+      }
+      if (writeRequest != null) {
+        this.totalWriteRequest = writeRequest.getTotalRequestCount();
+        this.writeRequestPerSec = writeRequest.getRequestPerSecond();
+      }
     }
 
     // Getters
@@ -171,6 +212,22 @@ public class HServerLoad implements Writ
       return storefileIndexSizeMB;
     }
 
+    public long getTotalReadRequest() {
+      return totalReadRequest;
+    }
+
+    public long getTotalWriteRequest() {
+      return totalWriteRequest;
+    }
+
+    public int getReadRequestPerSec() {
+      return readRequestPerSec;
+    }
+
+    public int getWriteRequestPerSec() {
+      return writeRequestPerSec;
+    }
+
     // Setters
 
     /**
@@ -219,6 +276,10 @@ public class HServerLoad implements Writ
       this.storefileSizeMB = in.readInt();
       this.memstoreSizeMB = in.readInt();
       this.storefileIndexSizeMB = in.readInt();
+      this.totalReadRequest = in.readLong();
+      this.readRequestPerSec = in.readInt();
+      this.totalWriteRequest = in.readLong();
+      this.writeRequestPerSec = in.readInt();
     }
 
     public void write(DataOutput out) throws IOException {
@@ -229,6 +290,10 @@ public class HServerLoad implements Writ
       out.writeInt(storefileSizeMB);
       out.writeInt(memstoreSizeMB);
       out.writeInt(storefileIndexSizeMB);
+      out.writeLong(this.totalReadRequest);
+      out.writeInt(this.readRequestPerSec);
+      out.writeLong(this.totalWriteRequest);
+      out.writeInt(this.writeRequestPerSec);
     }
 
     /**
@@ -261,6 +326,16 @@ public class HServerLoad implements Writ
         sb = Strings.appendKeyValue(sb, "totalStaticBloomSizeKB",
           Integer.valueOf(this.totalStaticBloomSizeKB));
       }
+
+      if (this.totalReadRequest != 0) {
+        sb = Strings.appendKeyValue(sb, "totalReadRequest",
+          Long.valueOf(this.totalReadRequest));
+      }
+
+      if (this.totalWriteRequest != 0) {
+        sb = Strings.appendKeyValue(sb, "totalWriteRequest",
+          Long.valueOf(this.totalWriteRequest));
+      }
       return sb.toString();
     }
   }

Added: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/metrics/RequestMetrics.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/metrics/RequestMetrics.java?rev=1293043&view=auto
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/metrics/RequestMetrics.java
(added)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/metrics/RequestMetrics.java
Fri Feb 24 00:50:31 2012
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.metrics;
+
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+
+/**
+ * The RequestMetrics is to keep track of total request count metrics
+ * as well as the requests per second
+ *
+ */
+public class RequestMetrics {
+  private long totalRequestCount = 0;
+  private long lastTotalRequestCount = 0;
+  private long lastUpdateTimeStamp = 0;
+  private int requestPerSecond = 0;
+  private static final long INTERVAL = 1000;
+
+  public RequestMetrics() {
+    this.lastUpdateTimeStamp = EnvironmentEdgeManager.currentTimeMillis();
+  }
+
+  public synchronized long getTotalRequestCount() {
+    return totalRequestCount;
+  }
+
+  public synchronized void incrTotalRequstCount(long incr) {
+    this.totalRequestCount += incr;
+  }
+
+  public synchronized void incrTotalRequstCount() {
+    this.totalRequestCount ++;
+  }
+
+  /**
+   * @return requests per second
+   */
+  public synchronized int getRequestPerSecond() {
+    long interval = EnvironmentEdgeManager.currentTimeMillis()
+      - lastUpdateTimeStamp;
+    if (interval == 0)
+      interval = 1;
+
+    if (interval >= INTERVAL) {
+      // update the request per second if the interval is more than one second
+      int sec = (int) (interval / INTERVAL);
+      long requsts = this.totalRequestCount - this.lastTotalRequestCount;
+      requestPerSecond =  (int) (requsts / sec);
+
+      //update the last updated time stamp and last total request count
+      this.lastTotalRequestCount = this.totalRequestCount;
+      this.lastUpdateTimeStamp = EnvironmentEdgeManager.currentTimeMillis();
+    }
+
+    return requestPerSecond;
+  }
+
+}
\ No newline at end of file

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1293043&r1=1293042&r2=1293043&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
(original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
Fri Feb 24 00:50:31 2012
@@ -31,7 +31,6 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.NavigableSet;
@@ -67,12 +66,12 @@ import org.apache.hadoop.hbase.DroppedSn
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.NotServingRegionException;
 import org.apache.hadoop.hbase.UnknownScannerException;
-import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.Put;
@@ -86,6 +85,7 @@ import org.apache.hadoop.hbase.io.Refere
 import org.apache.hadoop.hbase.io.hfile.BlockCache;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
+import org.apache.hadoop.hbase.metrics.RequestMetrics;
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
 import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
@@ -178,6 +178,8 @@ public class HRegion implements HeapSize
   // private byte [] name = null;
 
   protected final AtomicLong memstoreSize = new AtomicLong(0);
+  private RequestMetrics readRequests = null;
+  private RequestMetrics writeRequests = null;
 
   private HRegionServer regionServer = null;
 
@@ -473,6 +475,9 @@ public class HRegion implements HeapSize
     this.blockingMemStoreSize = this.memstoreFlushSize *
       conf.getLong("hbase.hregion.memstore.block.multiplier", 2);
     this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
+
+    this.readRequests =new RequestMetrics();
+    this.writeRequests =new RequestMetrics();
   }
 
   /**
@@ -1700,6 +1705,8 @@ public class HRegion implements HeapSize
   public void delete(Map<byte[], List<KeyValue>> familyMap, boolean writeToWAL)
   throws IOException {
     long now = EnvironmentEdgeManager.currentTimeMillis();
+    this.writeRequests.incrTotalRequstCount();
+
     byte [] byteNow = Bytes.toBytes(now);
     boolean flush = false;
 
@@ -1822,6 +1829,7 @@ public class HRegion implements HeapSize
    */
   public void put(Put put, Integer lockid, boolean writeToWAL)
   throws IOException {
+    this.writeRequests.incrTotalRequstCount();
     checkReadOnly();
 
     // Do a rough check that we have resources to accept a write.  The check is
@@ -1894,6 +1902,7 @@ public class HRegion implements HeapSize
    * @throws IOException
    */
   public OperationStatusCode[] put(Pair<Put, Integer>[] putsAndLocks) throws IOException
{
+    this.writeRequests.incrTotalRequstCount();
     BatchOperationInProgress<Pair<Put, Integer>> batchOp =
       new BatchOperationInProgress<Pair<Put,Integer>>(putsAndLocks);
 
@@ -2940,6 +2949,7 @@ public class HRegion implements HeapSize
     @Override
     public synchronized boolean next(List<KeyValue> outResults, int limit,
         String metric) throws IOException {
+      readRequests.incrTotalRequstCount();
       if (this.filterClosed) {
         throw new UnknownScannerException("Scanner was closed (timed out?) " +
             "after we renewed it. Could be caused by a very slow scanner " +
@@ -3595,7 +3605,7 @@ public class HRegion implements HeapSize
    */
   private List<KeyValue> get(final Get get) throws IOException {
     long now = EnvironmentEdgeManager.currentTimeMillis();
-
+    readRequests.incrTotalRequstCount();
     Scan scan = new Scan(get);
 
     List<KeyValue> results = new ArrayList<KeyValue>();
@@ -3633,6 +3643,7 @@ public class HRegion implements HeapSize
   throws IOException {
     // to be used for metrics
     long before = EnvironmentEdgeManager.currentTimeMillis();
+    this.writeRequests.incrTotalRequstCount();
 
     checkRow(row);
     boolean flush = false;
@@ -3713,7 +3724,7 @@ public class HRegion implements HeapSize
 
   public static final long FIXED_OVERHEAD = ClassSize.align(
       (5 * Bytes.SIZEOF_LONG) + 2 * ClassSize.ARRAY +
-      (25 * ClassSize.REFERENCE) + ClassSize.OBJECT + Bytes.SIZEOF_INT);
+      (27 * ClassSize.REFERENCE) + ClassSize.OBJECT + Bytes.SIZEOF_INT);
 
   public static final long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD +
       ClassSize.OBJECT + (2 * ClassSize.ATOMIC_BOOLEAN) +
@@ -3867,6 +3878,14 @@ public class HRegion implements HeapSize
     return false;
   }
 
+  public RequestMetrics getReadRequest() {
+    return this.readRequests;
+  }
+
+  public RequestMetrics getWriteRequest() {
+    return this.writeRequests;
+  }
+
   /**
    * A mocked list implementaion - discards all updates.
    */

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=1293043&r1=1293042&r2=1293043&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
(original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
Fri Feb 24 00:50:31 2012
@@ -38,13 +38,13 @@ import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Random;
 import java.util.Set;
 import java.util.SortedMap;
 import java.util.SortedSet;
 import java.util.TreeMap;
 import java.util.TreeSet;
-import java.util.Map.Entry;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.LinkedBlockingQueue;
@@ -68,7 +68,9 @@ import org.apache.hadoop.hbase.Abortable
 import org.apache.hadoop.hbase.Chore;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
 import org.apache.hadoop.hbase.HMsg;
+import org.apache.hadoop.hbase.HMsg.Type;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HRegionLocation;
 import org.apache.hadoop.hbase.HServerAddress;
@@ -78,15 +80,13 @@ import org.apache.hadoop.hbase.HTableDes
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.LeaseListener;
 import org.apache.hadoop.hbase.Leases;
+import org.apache.hadoop.hbase.Leases.LeaseStillHeldException;
 import org.apache.hadoop.hbase.LocalHBaseCluster;
 import org.apache.hadoop.hbase.NotServingRegionException;
 import org.apache.hadoop.hbase.RemoteExceptionHandler;
 import org.apache.hadoop.hbase.UnknownRowLockException;
 import org.apache.hadoop.hbase.UnknownScannerException;
 import org.apache.hadoop.hbase.YouAreDeadException;
-import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
-import org.apache.hadoop.hbase.HMsg.Type;
-import org.apache.hadoop.hbase.Leases.LeaseStillHeldException;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.MultiPut;
@@ -877,7 +877,8 @@ public class HRegionServer implements HR
     }
     return new HServerLoad.RegionLoad(name, stores, storefiles,
       storefileSizeMB, memstoreSizeMB, storefileIndexSizeMB, rootIndexSizeKB,
-      totalStaticIndexSizeKB, totalStaticBloomSizeKB);
+      totalStaticIndexSizeKB, totalStaticBloomSizeKB, r.getReadRequest(),
+      r.getWriteRequest());
   }
 
   /**

Modified: hbase/branches/0.89-fb/src/main/resources/hbase-webapps/regionserver/regionserver.jsp
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/resources/hbase-webapps/regionserver/regionserver.jsp?rev=1293043&r1=1293042&r2=1293043&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/resources/hbase-webapps/regionserver/regionserver.jsp
(original)
+++ hbase/branches/0.89-fb/src/main/resources/hbase-webapps/regionserver/regionserver.jsp
Fri Feb 24 00:50:31 2012
@@ -48,15 +48,24 @@
 <h2>Online Regions</h2>
 <% if (onlineRegions != null && onlineRegions.size() > 0) { %>
 <table>
-<tr><th>Region Name</th><th>Start Key</th><th>End Key</th><th>Metrics</th></tr>
+<tr><th>Region Name</th>
+	<th>Read Requests/sec</th>
+	<th>Write Requests/sec</th>
+	<th>Start Key</th>
+	<th>End Key</th><th>Metrics</th></tr>
 <%   for (HRegionInfo r: onlineRegions) {
         HServerLoad.RegionLoad load = regionServer.createRegionLoad(r.getRegionName());
  %>
 <tr><td><%= r.getRegionNameAsString() %></td>
-    <td><%= Bytes.toStringBinary(r.getStartKey()) %></td><td><%=
Bytes.toStringBinary(r.getEndKey()) %></td>
+	<td><%= load.getReadRequestPerSec() %></td>
+	<td><%= load.getWriteRequestPerSec() %></td>
+    <td><%= Bytes.toStringBinary(r.getStartKey()) %></td>
+	<td><%= Bytes.toStringBinary(r.getEndKey()) %></td>
     <td><%= load.toString() %></td>
-    </tr>
-<%   } %>
+</tr>
+<%
+     }
+%>
 </table>
 <p>Region names are made of the containing table's name, a comma,
 the start key, a comma, and a randomly generated region id.  To illustrate,



Mime
View raw message