Return-Path: X-Original-To: apmail-hbase-commits-archive@www.apache.org Delivered-To: apmail-hbase-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 45D62D391 for ; Tue, 25 Sep 2012 12:52:34 +0000 (UTC) Received: (qmail 93449 invoked by uid 500); 25 Sep 2012 12:52:34 -0000 Delivered-To: apmail-hbase-commits-archive@hbase.apache.org Received: (qmail 93126 invoked by uid 500); 25 Sep 2012 12:52:27 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 93110 invoked by uid 99); 25 Sep 2012 12:52:27 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 25 Sep 2012 12:52:27 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 25 Sep 2012 12:52:22 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id C865823888E3 for ; Tue, 25 Sep 2012 12:51:37 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1389841 - in /hbase/branches/0.89-fb/src: main/java/org/apache/hadoop/hbase/regionserver/ main/java/org/apache/hadoop/hbase/regionserver/metrics/ test/java/org/apache/hadoop/hbase/regionserver/ Date: Tue, 25 Sep 2012 12:51:37 -0000 To: commits@hbase.apache.org From: mbautin@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120925125137.C865823888E3@eris.apache.org> Author: mbautin Date: Tue Sep 25 12:51:37 2012 New Revision: 1389841 URL: http://svn.apache.org/viewvc?rev=1389841&view=rev Log: [HBASE-6872] Number of records read/written per second Author: adela Summary: as in title Test Plan: check NUM READS and NUM WRITES in testMultipleRegions() in TestRegionServerMetrics.java Reviewers: liyintang Reviewed By: liyintang Differential Revision: https://phabricator.fb.com/D556908 Task ID: 1149883 Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=1389841&r1=1389840&r2=1389841&view=diff ============================================================================== --- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original) +++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Tue Sep 25 12:51:37 2012 @@ -53,9 +53,7 @@ import java.util.concurrent.ExecutorServ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadFactory; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -243,8 +241,8 @@ public class HRegionServer implements HR // Leases private Leases leases; - // Request counter - private volatile AtomicInteger requestCount = new AtomicInteger(); + private volatile AtomicInteger numReads = new AtomicInteger(); + private volatile AtomicInteger numWrites = new AtomicInteger(); // Info server. Default access so can be used by unit tests. REGIONSERVER // is name of the webapp and the attribute name used stuffing this instance @@ -639,7 +637,7 @@ public class HRegionServer implements HR doMetrics(); MemoryUsage memory = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage(); - HServerLoad hsl = new HServerLoad(requestCount.get(), + HServerLoad hsl = new HServerLoad((numReads.get() + numWrites.get()), (int)(memory.getUsed()/1024/1024), (int)(memory.getMax()/1024/1024)); for (HRegion r: onlineRegions.values()) { @@ -647,7 +645,8 @@ public class HRegionServer implements HR } // XXX add a field in serverInfo to report to fsOK to master? this.serverInfo.setLoad(hsl); - this.requestCount.set(0); + numReads.set(0); + numWrites.set(0); addOutboundMsgs(outboundMessages); HMsg msgs[] = this.hbaseMaster.regionServerReport( serverInfo, outboundMessages.toArray(EMPTY_HMSG_ARRAY), @@ -979,6 +978,8 @@ public class HRegionServer implements HR this.dynamicMetrics = RegionServerDynamicMetrics.newInstance(this); startServiceThreads(); isOnline = true; + this.numReads.set(0); + this.numWrites.set(0); // Create the thread for the ThriftServer. // NOTE this defaults to FALSE so you have to enable it in conf @@ -1298,8 +1299,13 @@ public class HRegionServer implements HR } protected void metrics() { + int numReads = this.numReads.get(); + int numWrites = this.numWrites.get(); + this.metrics.regions.set(this.onlineRegions.size()); - this.metrics.incrementRequests(this.requestCount.get()); + this.metrics.incrementRequests(numReads + numWrites); + this.metrics.numReads.inc(numReads); + this.metrics.numWrites.inc(numWrites); // Is this too expensive every three seconds getting a lock on onlineRegions // and then per store carried? Can I make metrics be sloppier and avoid // the synchronizations? @@ -1715,7 +1721,6 @@ public class HRegionServer implements HR boolean znodeWritten = false; while(!stopRequested.get()) { try { - this.requestCount.set(0); MemoryUsage memory = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage(); HServerLoad hsl = new HServerLoad(0, (int)memory.getUsed()/1024/1024, @@ -2232,7 +2237,7 @@ public class HRegionServer implements HR @Override public HRegionInfo getRegionInfo(final byte [] regionName) throws NotServingRegionException { - requestCount.incrementAndGet(); + numReads.incrementAndGet(); return getRegion(regionName).getRegionInfo(); } @@ -2242,12 +2247,11 @@ public class HRegionServer implements HR final byte [] row, final byte [] family) throws IOException { checkOpen(); - requestCount.incrementAndGet(); + numReads.incrementAndGet(); try { // locate the region we're operating on HRegion region = getRegion(regionName); // ask the region for all the data - Result r = region.getClosestRowBefore(row, family); return r; } catch (Throwable t) { @@ -2259,7 +2263,7 @@ public class HRegionServer implements HR @Override public Result get(byte [] regionName, Get get) throws IOException { checkOpen(); - requestCount.incrementAndGet(); + numReads.incrementAndGet(); try { HRegion region = getRegion(regionName); return region.get(get, getLockFromId(get.getLockId())); @@ -2272,7 +2276,7 @@ public class HRegionServer implements HR public Result[] get(byte[] regionName, List gets) throws IOException { checkOpen(); - requestCount.addAndGet(gets.size()); + numReads.addAndGet(gets.size()); Result[] rets = new Result[gets.size()]; try { HRegion region = getRegion(regionName); @@ -2295,14 +2299,14 @@ public class HRegionServer implements HR throw new IOException("Invalid arguments to atomicMutation " + "regionName is null"); } - requestCount.incrementAndGet(); + numWrites.incrementAndGet(); try { HRegion region = getRegion(regionName); if (!region.getRegionInfo().isMetaTable()) { this.cacheFlusher.reclaimMemStoreMemory(); } for (RowMutations arm: armList) { - this.requestCount.incrementAndGet(); + numWrites.incrementAndGet(); region.mutateRow(arm); } } catch (Throwable t) { @@ -2319,7 +2323,7 @@ public class HRegionServer implements HR @Override public boolean exists(byte [] regionName, Get get) throws IOException { checkOpen(); - requestCount.incrementAndGet(); + numReads.incrementAndGet(); try { HRegion region = getRegion(regionName); Result r = region.get(get, getLockFromId(get.getLockId())); @@ -2336,7 +2340,7 @@ public class HRegionServer implements HR throw new IllegalArgumentException("update has null row"); checkOpen(); - this.requestCount.incrementAndGet(); + numWrites.incrementAndGet(); HRegion region = getRegion(regionName); try { if (!region.getRegionInfo().isMetaTable()) { @@ -2376,7 +2380,7 @@ public class HRegionServer implements HR opWithLocks[i++] = new Pair(p, lock); } - this.requestCount.addAndGet(mutations.size()); + numWrites.addAndGet(mutations.size()); OperationStatusCode[] codes = region.batchMutateWithLocks(opWithLocks, methodName); for (i = 0; i < codes.length; i++) { @@ -2393,7 +2397,7 @@ public class HRegionServer implements HR final byte [] family, final byte [] qualifier, final byte [] value, final Writable w, Integer lock) throws IOException { checkOpen(); - this.requestCount.incrementAndGet(); + numWrites.incrementAndGet(); HRegion region = getRegion(regionName); try { if (!region.getRegionInfo().isMetaTable()) { @@ -2462,7 +2466,7 @@ public class HRegionServer implements HR if (npe != null) { throw new IOException("Invalid arguments to openScanner", npe); } - requestCount.incrementAndGet(); + numReads.incrementAndGet(); try { HRegion r = getRegion(regionName); return addScanner(r.getScanner(scan)); @@ -2545,7 +2549,6 @@ public class HRegionServer implements HR List results = new ArrayList(nbRows); long currentScanResultSize = 0; List values = new ArrayList(); - int i = 0; for (; i < nbRows && currentScanResultSize < maxScannerResultSize; i++) { // Collect values to be returned here @@ -2563,8 +2566,7 @@ public class HRegionServer implements HR } values.clear(); } - requestCount.addAndGet(i); - + numReads.addAndGet(i); // Below is an ugly hack where we cast the InternalScanner to be a // HRegion.RegionScanner. The alternative is to change InternalScanner // interface but its used everywhere whereas we just need a bit of info @@ -2586,7 +2588,7 @@ public class HRegionServer implements HR public void close(final long scannerId) throws IOException { try { checkOpen(); - requestCount.incrementAndGet(); + numReads.incrementAndGet(); String scannerName = String.valueOf(scannerId); InternalScanner s = scanners.remove(scannerName); if (s != null) { @@ -2632,7 +2634,7 @@ public class HRegionServer implements HR checkOpen(); try { boolean writeToWAL = delete.getWriteToWAL(); - this.requestCount.incrementAndGet(); + numWrites.incrementAndGet(); HRegion region = getRegion(regionName); if (!region.getRegionInfo().isMetaTable()) { this.cacheFlusher.reclaimMemStoreMemory(); @@ -2665,7 +2667,7 @@ public class HRegionServer implements HR io.initCause(npe); throw io; } - requestCount.incrementAndGet(); + numReads.incrementAndGet(); try { HRegion region = getRegion(regionName); Integer r = region.obtainRowLock(row); @@ -2724,7 +2726,7 @@ public class HRegionServer implements HR io.initCause(npe); throw io; } - requestCount.incrementAndGet(); + numReads.incrementAndGet(); try { HRegion region = getRegion(regionName); String lockName = String.valueOf(lockId); @@ -3001,9 +3003,14 @@ public class HRegionServer implements HR return onlineRegions.get(Bytes.mapKey(regionName)); } - /** @return the request count */ - public AtomicInteger getRequestCount() { - return this.requestCount; + /** @return the number of reads */ + public AtomicInteger getNumReads() { + return this.numReads; + } + + /** @return the number of writes */ + public AtomicInteger getNumWrites() { + return this.numWrites; } /** @return reference to FlushRequester */ @@ -3164,7 +3171,7 @@ public class HRegionServer implements HR throw new IOException("Invalid arguments to incrementColumnValue " + "regionName is null"); } - requestCount.incrementAndGet(); + numWrites.incrementAndGet(); try { HRegion region = getRegion(regionName); long retval = region.incrementColumnValue(row, family, qualifier, amount, @@ -3514,5 +3521,4 @@ public class HRegionServer implements HR } return 0; } - } Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java?rev=1389841&r1=1389840&r2=1389841&view=diff ============================================================================== --- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java (original) +++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java Tue Sep 25 12:51:37 2012 @@ -38,6 +38,8 @@ import org.apache.hadoop.metrics.jvm.Jvm import org.apache.hadoop.metrics.util.MetricsIntValue; import org.apache.hadoop.metrics.util.MetricsLongValue; import org.apache.hadoop.metrics.util.MetricsRegistry; +import org.apache.hadoop.metrics.util.MetricsTimeVaryingInt; +import org.apache.hadoop.metrics.util.MetricsTimeVaryingLong; import org.apache.hadoop.metrics.util.MetricsTimeVaryingRate; import java.io.IOException; @@ -200,6 +202,12 @@ public class RegionServerMetrics impleme public final MetricsTimeVaryingRate mvccWaitTime = new MetricsTimeVaryingRate("mvccWait", registry); + public final MetricsRate numReads = + new MetricsRate("numReads", registry); + + public final MetricsRate numWrites = + new MetricsRate("numWrites", registry); + /** * time each scheduled compaction takes */ @@ -286,6 +294,8 @@ public class RegionServerMetrics impleme this.blockCacheEvictedMultiCount.pushMetric(this.metricsRecord); this.blockCacheEvictedMemoryCount.pushMetric(this.metricsRecord); this.blockCacheHitRatio.pushMetric(this.metricsRecord); + this.numReads.pushMetric(this.metricsRecord); + this.numWrites.pushMetric(this.metricsRecord); // Be careful. Here is code for MTVR from up in hadoop: // public synchronized void inc(final int numOps, final long time) { @@ -422,6 +432,11 @@ public class RegionServerMetrics impleme Integer.valueOf(this.memstoreSizeMB.get())); sb = Strings.appendKeyValue(sb, "compactionQueueSize", Integer.valueOf(this.compactionQueueSize.get())); + sb = Strings.appendKeyValue(sb, "numWrites", + Float.valueOf(this.numWrites.getPreviousIntervalValue())); + sb = Strings.appendKeyValue(sb, "numReads", + Float.valueOf(this.numReads.getPreviousIntervalValue())); + // Duplicate from jvmmetrics because metrics are private there so // inaccessible. MemoryUsage memory = Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java?rev=1389841&r1=1389840&r2=1389841&view=diff ============================================================================== --- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java (original) +++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java Tue Sep 25 12:51:37 2012 @@ -21,9 +21,9 @@ package org.apache.hadoop.hbase.regionse import static org.junit.Assert.assertEquals; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Map; -import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -34,13 +34,12 @@ import org.apache.hadoop.hbase.client.HB import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics. StoreMetricType; import org.apache.hadoop.hbase.util.Bytes; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -207,4 +206,21 @@ public class TestRegionServerMetrics { assertSizeMetric(tableName, cfs, new int[] {kvLength, kvLength, kvLength, kvLength}); } + + @Test + public void testNumReadsAndWrites() throws IOException, InterruptedException{ + TEST_UTIL.createRandomTable( + "NumReadsWritesTest", + Arrays.asList(FAMILIES), + MAX_VERSIONS, NUM_COLS_PER_ROW, NUM_FLUSHES, NUM_REGIONS, 1000); + final HRegionServer rs = + TEST_UTIL.getMiniHBaseCluster().getRegionServer(0); + rs.doMetrics(); + for (HRegion r : rs.getOnlineRegions()) { + Get g = new Get(new byte[]{}); + rs.get(r.getRegionName(), g); + } + Assert.assertEquals(rs.getOnlineRegions().size(), rs.getNumReads().get()); + Assert.assertEquals(rs.getNumWrites().get(), 0); + } }