hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mbau...@apache.org
Subject svn commit: r1389841 - in /hbase/branches/0.89-fb/src: main/java/org/apache/hadoop/hbase/regionserver/ main/java/org/apache/hadoop/hbase/regionserver/metrics/ test/java/org/apache/hadoop/hbase/regionserver/
Date Tue, 25 Sep 2012 12:51:37 GMT
Author: mbautin
Date: Tue Sep 25 12:51:37 2012
New Revision: 1389841

URL: http://svn.apache.org/viewvc?rev=1389841&view=rev
Log:
[HBASE-6872] Number of records read/written per second

Author: adela

Summary: as in title

Test Plan: check NUM READS and NUM WRITES in testMultipleRegions() in TestRegionServerMetrics.java

Reviewers: liyintang

Reviewed By: liyintang

Differential Revision: https://phabricator.fb.com/D556908

Task ID: 1149883

Modified:
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=1389841&r1=1389840&r2=1389841&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
(original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
Tue Sep 25 12:51:37 2012
@@ -53,9 +53,7 @@ import java.util.concurrent.ExecutorServ
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.SynchronousQueue;
 import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -243,8 +241,8 @@ public class HRegionServer implements HR
   // Leases
   private Leases leases;
 
-  // Request counter
-  private volatile AtomicInteger requestCount = new AtomicInteger();
+  private volatile AtomicInteger numReads = new AtomicInteger();
+  private volatile AtomicInteger numWrites = new AtomicInteger();
 
   // Info server.  Default access so can be used by unit tests.  REGIONSERVER
   // is name of the webapp and the attribute name used stuffing this instance
@@ -639,7 +637,7 @@ public class HRegionServer implements HR
             doMetrics();
             MemoryUsage memory =
               ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
-            HServerLoad hsl = new HServerLoad(requestCount.get(),
+            HServerLoad hsl = new HServerLoad((numReads.get() + numWrites.get()),
               (int)(memory.getUsed()/1024/1024),
               (int)(memory.getMax()/1024/1024));
             for (HRegion r: onlineRegions.values()) {
@@ -647,7 +645,8 @@ public class HRegionServer implements HR
             }
             // XXX add a field in serverInfo to report to fsOK to master?
             this.serverInfo.setLoad(hsl);
-            this.requestCount.set(0);
+            numReads.set(0);
+            numWrites.set(0);
             addOutboundMsgs(outboundMessages);
             HMsg msgs[] = this.hbaseMaster.regionServerReport(
               serverInfo, outboundMessages.toArray(EMPTY_HMSG_ARRAY),
@@ -979,6 +978,8 @@ public class HRegionServer implements HR
       this.dynamicMetrics = RegionServerDynamicMetrics.newInstance(this);
       startServiceThreads();
       isOnline = true;
+      this.numReads.set(0);
+      this.numWrites.set(0);
 
       // Create the thread for the ThriftServer.
       // NOTE this defaults to FALSE so you have to enable it in conf
@@ -1298,8 +1299,13 @@ public class HRegionServer implements HR
   }
 
   protected void metrics() {
+    int numReads = this.numReads.get();
+    int numWrites = this.numWrites.get();
+
     this.metrics.regions.set(this.onlineRegions.size());
-    this.metrics.incrementRequests(this.requestCount.get());
+    this.metrics.incrementRequests(numReads + numWrites);
+    this.metrics.numReads.inc(numReads);
+    this.metrics.numWrites.inc(numWrites);
     // Is this too expensive every three seconds getting a lock on onlineRegions
     // and then per store carried?  Can I make metrics be sloppier and avoid
     // the synchronizations?
@@ -1715,7 +1721,6 @@ public class HRegionServer implements HR
     boolean znodeWritten = false;
     while(!stopRequested.get()) {
       try {
-        this.requestCount.set(0);
         MemoryUsage memory =
           ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
         HServerLoad hsl = new HServerLoad(0, (int)memory.getUsed()/1024/1024,
@@ -2232,7 +2237,7 @@ public class HRegionServer implements HR
   @Override
   public HRegionInfo getRegionInfo(final byte [] regionName)
   throws NotServingRegionException {
-    requestCount.incrementAndGet();
+    numReads.incrementAndGet();
     return getRegion(regionName).getRegionInfo();
   }
 
@@ -2242,12 +2247,11 @@ public class HRegionServer implements HR
     final byte [] row, final byte [] family)
   throws IOException {
     checkOpen();
-    requestCount.incrementAndGet();
+    numReads.incrementAndGet();
     try {
       // locate the region we're operating on
       HRegion region = getRegion(regionName);
       // ask the region for all the data
-
       Result r = region.getClosestRowBefore(row, family);
       return r;
     } catch (Throwable t) {
@@ -2259,7 +2263,7 @@ public class HRegionServer implements HR
   @Override
   public Result get(byte [] regionName, Get get) throws IOException {
     checkOpen();
-    requestCount.incrementAndGet();
+    numReads.incrementAndGet();
     try {
       HRegion region = getRegion(regionName);
       return region.get(get, getLockFromId(get.getLockId()));
@@ -2272,7 +2276,7 @@ public class HRegionServer implements HR
   public Result[] get(byte[] regionName, List<Get> gets)
       throws IOException {
     checkOpen();
-    requestCount.addAndGet(gets.size());
+    numReads.addAndGet(gets.size());
     Result[] rets = new Result[gets.size()];
     try {
       HRegion region = getRegion(regionName);
@@ -2295,14 +2299,14 @@ public class HRegionServer implements HR
       throw new IOException("Invalid arguments to atomicMutation " +
       "regionName is null");
     }
-    requestCount.incrementAndGet();
+    numWrites.incrementAndGet();
     try {
       HRegion region = getRegion(regionName);
       if (!region.getRegionInfo().isMetaTable()) {
         this.cacheFlusher.reclaimMemStoreMemory();
       }
       for (RowMutations arm: armList) {
-        this.requestCount.incrementAndGet();
+        numWrites.incrementAndGet();
         region.mutateRow(arm);
       }
     } catch (Throwable t) {
@@ -2319,7 +2323,7 @@ public class HRegionServer implements HR
   @Override
   public boolean exists(byte [] regionName, Get get) throws IOException {
     checkOpen();
-    requestCount.incrementAndGet();
+    numReads.incrementAndGet();
     try {
       HRegion region = getRegion(regionName);
       Result r = region.get(get, getLockFromId(get.getLockId()));
@@ -2336,7 +2340,7 @@ public class HRegionServer implements HR
       throw new IllegalArgumentException("update has null row");
 
     checkOpen();
-    this.requestCount.incrementAndGet();
+    numWrites.incrementAndGet();
     HRegion region = getRegion(regionName);
     try {
       if (!region.getRegionInfo().isMetaTable()) {
@@ -2376,7 +2380,7 @@ public class HRegionServer implements HR
         opWithLocks[i++] = new Pair<Mutation, Integer>(p, lock);
       }
 
-      this.requestCount.addAndGet(mutations.size());
+      numWrites.addAndGet(mutations.size());
       OperationStatusCode[] codes = region.batchMutateWithLocks(opWithLocks,
           methodName);
       for (i = 0; i < codes.length; i++) {
@@ -2393,7 +2397,7 @@ public class HRegionServer implements HR
       final byte [] family, final byte [] qualifier, final byte [] value,
       final Writable w, Integer lock) throws IOException {
     checkOpen();
-    this.requestCount.incrementAndGet();
+    numWrites.incrementAndGet();
     HRegion region = getRegion(regionName);
     try {
       if (!region.getRegionInfo().isMetaTable()) {
@@ -2462,7 +2466,7 @@ public class HRegionServer implements HR
     if (npe != null) {
       throw new IOException("Invalid arguments to openScanner", npe);
     }
-    requestCount.incrementAndGet();
+    numReads.incrementAndGet();
     try {
       HRegion r = getRegion(regionName);
       return addScanner(r.getScanner(scan));
@@ -2545,7 +2549,6 @@ public class HRegionServer implements HR
       List<Result> results = new ArrayList<Result>(nbRows);
       long currentScanResultSize = 0;
       List<KeyValue> values = new ArrayList<KeyValue>();
-
       int i = 0;
       for (; i < nbRows && currentScanResultSize < maxScannerResultSize; i++)
{
         // Collect values to be returned here
@@ -2563,8 +2566,7 @@ public class HRegionServer implements HR
         }
         values.clear();
       }
-      requestCount.addAndGet(i);
-
+      numReads.addAndGet(i);
       // Below is an ugly hack where we cast the InternalScanner to be a
       // HRegion.RegionScanner.  The alternative is to change InternalScanner
       // interface but its used everywhere whereas we just need a bit of info
@@ -2586,7 +2588,7 @@ public class HRegionServer implements HR
   public void close(final long scannerId) throws IOException {
     try {
       checkOpen();
-      requestCount.incrementAndGet();
+      numReads.incrementAndGet();
       String scannerName = String.valueOf(scannerId);
       InternalScanner s = scanners.remove(scannerName);
       if (s != null) {
@@ -2632,7 +2634,7 @@ public class HRegionServer implements HR
     checkOpen();
     try {
       boolean writeToWAL = delete.getWriteToWAL();
-      this.requestCount.incrementAndGet();
+      numWrites.incrementAndGet();
       HRegion region = getRegion(regionName);
       if (!region.getRegionInfo().isMetaTable()) {
         this.cacheFlusher.reclaimMemStoreMemory();
@@ -2665,7 +2667,7 @@ public class HRegionServer implements HR
       io.initCause(npe);
       throw io;
     }
-    requestCount.incrementAndGet();
+    numReads.incrementAndGet();
     try {
       HRegion region = getRegion(regionName);
       Integer r = region.obtainRowLock(row);
@@ -2724,7 +2726,7 @@ public class HRegionServer implements HR
       io.initCause(npe);
       throw io;
     }
-    requestCount.incrementAndGet();
+    numReads.incrementAndGet();
     try {
       HRegion region = getRegion(regionName);
       String lockName = String.valueOf(lockId);
@@ -3001,9 +3003,14 @@ public class HRegionServer implements HR
     return onlineRegions.get(Bytes.mapKey(regionName));
   }
 
-  /** @return the request count */
-  public AtomicInteger getRequestCount() {
-    return this.requestCount;
+  /** @return the number of reads */
+  public AtomicInteger getNumReads() {
+    return this.numReads;
+  }
+
+  /** @return the number of writes */
+  public AtomicInteger getNumWrites() {
+    return this.numWrites;
   }
 
   /** @return reference to FlushRequester */
@@ -3164,7 +3171,7 @@ public class HRegionServer implements HR
       throw new IOException("Invalid arguments to incrementColumnValue " +
       "regionName is null");
     }
-    requestCount.incrementAndGet();
+    numWrites.incrementAndGet();
     try {
       HRegion region = getRegion(regionName);
       long retval = region.incrementColumnValue(row, family, qualifier, amount,
@@ -3514,5 +3521,4 @@ public class HRegionServer implements HR
     }
     return 0;
   }
-
 }

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java?rev=1389841&r1=1389840&r2=1389841&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
(original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
Tue Sep 25 12:51:37 2012
@@ -38,6 +38,8 @@ import org.apache.hadoop.metrics.jvm.Jvm
 import org.apache.hadoop.metrics.util.MetricsIntValue;
 import org.apache.hadoop.metrics.util.MetricsLongValue;
 import org.apache.hadoop.metrics.util.MetricsRegistry;
+import org.apache.hadoop.metrics.util.MetricsTimeVaryingInt;
+import org.apache.hadoop.metrics.util.MetricsTimeVaryingLong;
 import org.apache.hadoop.metrics.util.MetricsTimeVaryingRate;
 
 import java.io.IOException;
@@ -200,6 +202,12 @@ public class RegionServerMetrics impleme
   public final MetricsTimeVaryingRate mvccWaitTime =
     new MetricsTimeVaryingRate("mvccWait", registry);
 
+  public final MetricsRate numReads =
+    new MetricsRate("numReads", registry);
+
+  public final MetricsRate numWrites =
+    new MetricsRate("numWrites", registry);
+
   /**
    * time each scheduled compaction takes
    */
@@ -286,6 +294,8 @@ public class RegionServerMetrics impleme
       this.blockCacheEvictedMultiCount.pushMetric(this.metricsRecord);
       this.blockCacheEvictedMemoryCount.pushMetric(this.metricsRecord);
       this.blockCacheHitRatio.pushMetric(this.metricsRecord);
+      this.numReads.pushMetric(this.metricsRecord);
+      this.numWrites.pushMetric(this.metricsRecord);
 
       // Be careful. Here is code for MTVR from up in hadoop:
       // public synchronized void inc(final int numOps, final long time) {
@@ -422,6 +432,11 @@ public class RegionServerMetrics impleme
       Integer.valueOf(this.memstoreSizeMB.get()));
     sb = Strings.appendKeyValue(sb, "compactionQueueSize",
       Integer.valueOf(this.compactionQueueSize.get()));
+    sb = Strings.appendKeyValue(sb, "numWrites",
+      Float.valueOf(this.numWrites.getPreviousIntervalValue()));
+    sb = Strings.appendKeyValue(sb, "numReads",
+      Float.valueOf(this.numReads.getPreviousIntervalValue()));
+
     // Duplicate from jvmmetrics because metrics are private there so
     // inaccessible.
     MemoryUsage memory =

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java?rev=1389841&r1=1389840&r2=1389841&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
(original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
Tue Sep 25 12:51:37 2012
@@ -21,9 +21,9 @@ package org.apache.hadoop.hbase.regionse
 import static org.junit.Assert.assertEquals;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Map;
-import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -34,13 +34,12 @@ import org.apache.hadoop.hbase.client.HB
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics.
     StoreMetricType;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.junit.After;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -207,4 +206,21 @@ public class TestRegionServerMetrics {
     assertSizeMetric(tableName, cfs,
         new int[] {kvLength, kvLength, kvLength, kvLength});
   }
+
+  @Test
+  public void testNumReadsAndWrites() throws IOException, InterruptedException{
+    TEST_UTIL.createRandomTable(
+        "NumReadsWritesTest",
+        Arrays.asList(FAMILIES),
+        MAX_VERSIONS, NUM_COLS_PER_ROW, NUM_FLUSHES, NUM_REGIONS, 1000);
+    final HRegionServer rs =
+        TEST_UTIL.getMiniHBaseCluster().getRegionServer(0);
+    rs.doMetrics();
+    for (HRegion r : rs.getOnlineRegions()) {
+      Get g = new Get(new byte[]{});
+      rs.get(r.getRegionName(), g);
+    }
+    Assert.assertEquals(rs.getOnlineRegions().size(), rs.getNumReads().get());
+    Assert.assertEquals(rs.getNumWrites().get(), 0);
+  }
 }



Mime
View raw message