Return-Path: X-Original-To: apmail-hbase-commits-archive@www.apache.org Delivered-To: apmail-hbase-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id B8125106CF for ; Sat, 15 Jun 2013 04:24:22 +0000 (UTC) Received: (qmail 88823 invoked by uid 500); 15 Jun 2013 04:24:22 -0000 Delivered-To: apmail-hbase-commits-archive@hbase.apache.org Received: (qmail 88611 invoked by uid 500); 15 Jun 2013 04:24:22 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 88604 invoked by uid 99); 15 Jun 2013 04:24:22 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 15 Jun 2013 04:24:22 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 15 Jun 2013 04:24:20 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 36AF42388A2C; Sat, 15 Jun 2013 04:24:00 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1493308 - in /hbase/branches/0.95: hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/ hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/ hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/... Date: Sat, 15 Jun 2013 04:23:59 -0000 To: commits@hbase.apache.org From: stack@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20130615042400.36AF42388A2C@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: stack Date: Sat Jun 15 04:23:59 2013 New Revision: 1493308 URL: http://svn.apache.org/r1493308 Log: HBASE-8689 Cover all mutations rather than only Put while reporting for mutations not writing to WAL Modified: hbase/branches/0.95/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java hbase/branches/0.95/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java hbase/branches/0.95/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java hbase/branches/0.95/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java Modified: hbase/branches/0.95/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java?rev=1493308&r1=1493307&r2=1493308&view=diff ============================================================================== --- hbase/branches/0.95/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java (original) +++ hbase/branches/0.95/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java Sat Jun 15 04:23:59 2013 @@ -145,10 +145,10 @@ public interface MetricsRegionServerSour static final String STATIC_BLOOM_SIZE = "staticBloomSize"; static final String STATIC_BLOOM_SIZE_DESC = "Uncompressed size of the static bloom filters."; - static final String NUMBER_OF_PUTS_WITHOUT_WAL = "putsWithoutWALCount"; - static final String NUMBER_OF_PUTS_WITHOUT_WAL_DESC = + static final String NUMBER_OF_MUTATIONS_WITHOUT_WAL = "mutationsWithoutWALCount"; + static final String NUMBER_OF_MUTATIONS_WITHOUT_WAL_DESC = "Number of mutations that have been sent by clients with the write ahead logging turned off."; - static final String DATA_SIZE_WITHOUT_WAL = "putsWithoutWALSize"; + static final String DATA_SIZE_WITHOUT_WAL = "mutationsWithoutWALSize"; static final String DATA_SIZE_WITHOUT_WAL_DESC = "Size of data that has been sent by clients with the write ahead logging turned off."; static final String PERCENT_FILES_LOCAL = "percentFilesLocal"; Modified: hbase/branches/0.95/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java?rev=1493308&r1=1493307&r2=1493308&view=diff ============================================================================== --- hbase/branches/0.95/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java (original) +++ hbase/branches/0.95/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java Sat Jun 15 04:23:59 2013 @@ -130,7 +130,7 @@ public interface MetricsRegionServerWrap /** * Number of mutations received with WAL explicitly turned off. */ - long getNumPutsWithoutWAL(); + long getNumMutationsWithoutWAL(); /** * Ammount of data in the memstore but not in the WAL because mutations explicitly had their Modified: hbase/branches/0.95/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java?rev=1493308&r1=1493307&r2=1493308&view=diff ============================================================================== --- hbase/branches/0.95/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java (original) +++ hbase/branches/0.95/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java Sat Jun 15 04:23:59 2013 @@ -164,9 +164,9 @@ public class MetricsRegionServerSourceIm .addGauge(STOREFILE_INDEX_SIZE, STOREFILE_INDEX_SIZE_DESC, rsWrap.getStoreFileIndexSize()) .addGauge(STATIC_INDEX_SIZE, STATIC_INDEX_SIZE_DESC, rsWrap.getTotalStaticIndexSize()) .addGauge(STATIC_BLOOM_SIZE, STATIC_BLOOM_SIZE_DESC, rsWrap.getTotalStaticBloomSize()) - .addGauge(NUMBER_OF_PUTS_WITHOUT_WAL, - NUMBER_OF_PUTS_WITHOUT_WAL_DESC, - rsWrap.getNumPutsWithoutWAL()) + .addGauge(NUMBER_OF_MUTATIONS_WITHOUT_WAL, + NUMBER_OF_MUTATIONS_WITHOUT_WAL_DESC, + rsWrap.getNumMutationsWithoutWAL()) .addGauge(DATA_SIZE_WITHOUT_WAL, DATA_SIZE_WITHOUT_WAL_DESC, rsWrap.getDataInMemoryWithoutWAL()) Modified: hbase/branches/0.95/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java?rev=1493308&r1=1493307&r2=1493308&view=diff ============================================================================== --- hbase/branches/0.95/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java (original) +++ hbase/branches/0.95/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java Sat Jun 15 04:23:59 2013 @@ -173,9 +173,10 @@ public class MetricsRegionServerSourceIm .addGauge(Interns.info(STATIC_INDEX_SIZE, STATIC_INDEX_SIZE_DESC), rsWrap.getTotalStaticIndexSize()) .addGauge(Interns.info(STATIC_BLOOM_SIZE, STATIC_BLOOM_SIZE_DESC), - rsWrap.getTotalStaticBloomSize()) - .addGauge(Interns.info(NUMBER_OF_PUTS_WITHOUT_WAL, NUMBER_OF_PUTS_WITHOUT_WAL_DESC), - rsWrap.getNumPutsWithoutWAL()) + rsWrap.getTotalStaticBloomSize()) + .addGauge( + Interns.info(NUMBER_OF_MUTATIONS_WITHOUT_WAL, NUMBER_OF_MUTATIONS_WITHOUT_WAL_DESC), + rsWrap.getNumMutationsWithoutWAL()) .addGauge(Interns.info(DATA_SIZE_WITHOUT_WAL, DATA_SIZE_WITHOUT_WAL_DESC), rsWrap.getDataInMemoryWithoutWAL()) .addGauge(Interns.info(PERCENT_FILES_LOCAL, PERCENT_FILES_LOCAL_DESC), Modified: hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1493308&r1=1493307&r2=1493308&view=diff ============================================================================== --- hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original) +++ hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java Sat Jun 15 04:23:59 2013 @@ -232,7 +232,7 @@ public class HRegion implements HeapSize public final AtomicLong memstoreSize = new AtomicLong(0); // Debug possible data loss due to WAL off - final Counter numPutsWithoutWAL = new Counter(); + final Counter numMutationsWithoutWAL = new Counter(); final Counter dataInMemoryWithoutWAL = new Counter(); // Debug why CAS operations are taking a while. @@ -502,7 +502,6 @@ public class HRegion implements HeapSize // When hbase.regionserver.optionallogflushinterval <= 0 , deferred log sync is disabled. this.deferredLogSyncDisabled = conf.getLong("hbase.regionserver.optionallogflushinterval", 1 * 1000) <= 0; - if (rsServices != null) { this.rsAccounting = this.rsServices.getRegionServerAccounting(); // don't initialize coprocessors if not running within a regionserver @@ -1367,8 +1366,8 @@ public class HRegion implements HeapSize status.setStatus("Running coprocessor pre-flush hooks"); coprocessorHost.preFlush(); } - if (numPutsWithoutWAL.get() > 0) { - numPutsWithoutWAL.set(0); + if (numMutationsWithoutWAL.get() > 0) { + numMutationsWithoutWAL.set(0); dataInMemoryWithoutWAL.set(0); } synchronized (writestate) { @@ -2224,9 +2223,7 @@ public class HRegion implements HeapSize durability = tmpDur; } if (tmpDur == Durability.SKIP_WAL) { - if (m instanceof Put) { - recordPutWithoutWal(m.getFamilyMap()); - } + recordMutationWithoutWal(m.getFamilyMap()); continue; } @@ -4869,10 +4866,11 @@ public class HRegion implements HeapSize // Using default cluster id, as this can only happen in the orginating // cluster. A slave cluster receives the final value (not the delta) // as a Put. - txid = this.log.appendNoSync(this.getRegionInfo(), - this.htableDescriptor.getName(), walEdits, - HConstants.DEFAULT_CLUSTER_ID, EnvironmentEdgeManager.currentTimeMillis(), - this.htableDescriptor); + txid = this.log.appendNoSync(this.getRegionInfo(), this.htableDescriptor.getName(), + walEdits, HConstants.DEFAULT_CLUSTER_ID, EnvironmentEdgeManager.currentTimeMillis(), + this.htableDescriptor); + } else { + recordMutationWithoutWal(append.getFamilyMap()); } //Actually write to Memstore now @@ -5017,8 +5015,9 @@ public class HRegion implements HeapSize txid = this.log.appendNoSync(this.getRegionInfo(), this.htableDescriptor.getName(), walEdits, HConstants.DEFAULT_CLUSTER_ID, EnvironmentEdgeManager.currentTimeMillis(), this.htableDescriptor); + } else { + recordMutationWithoutWal(increment.getFamilyMap()); } - //Actually write to Memstore now for (Map.Entry> entry : tempMemstore.entrySet()) { Store store = entry.getKey(); @@ -5445,22 +5444,22 @@ public class HRegion implements HeapSize * Update counters for numer of puts without wal and the size of possible data loss. * These information are exposed by the region server metrics. */ - private void recordPutWithoutWal(final Map> familyMap) { - numPutsWithoutWAL.increment(); - if (numPutsWithoutWAL.get() <= 1) { + private void recordMutationWithoutWal(final Map> familyMap) { + numMutationsWithoutWAL.increment(); + if (numMutationsWithoutWAL.get() <= 1) { LOG.info("writing data to region " + this + " with WAL disabled. Data may be lost in the event of a crash."); } - long putSize = 0; + long mutationSize = 0; for (List cells: familyMap.values()) { for (Cell cell : cells) { KeyValue kv = KeyValueUtil.ensureKeyValue(cell); - putSize += kv.getKeyLength() + kv.getValueLength(); + mutationSize += kv.getKeyLength() + kv.getValueLength(); } } - dataInMemoryWithoutWAL.add(putSize); + dataInMemoryWithoutWAL.add(mutationSize); } private void lock(final Lock lock) Modified: hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java?rev=1493308&r1=1493307&r2=1493308&view=diff ============================================================================== --- hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java (original) +++ hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java Sat Jun 15 04:23:59 2013 @@ -62,7 +62,7 @@ class MetricsRegionServerWrapperImpl private volatile long storefileIndexSize = 0; private volatile long totalStaticIndexSize = 0; private volatile long totalStaticBloomSize = 0; - private volatile long numPutsWithoutWAL = 0; + private volatile long numMutationsWithoutWAL = 0; private volatile long dataInMemoryWithoutWAL = 0; private volatile int percentFileLocal = 0; @@ -292,8 +292,8 @@ class MetricsRegionServerWrapperImpl } @Override - public long getNumPutsWithoutWAL() { - return numPutsWithoutWAL; + public long getNumMutationsWithoutWAL() { + return numMutationsWithoutWAL; } @Override @@ -344,13 +344,13 @@ class MetricsRegionServerWrapperImpl long tempStorefileIndexSize = 0; long tempTotalStaticIndexSize = 0; long tempTotalStaticBloomSize = 0; - long tempNumPutsWithoutWAL = 0; + long tempNumMutationsWithoutWAL = 0; long tempDataInMemoryWithoutWAL = 0; int tempPercentFileLocal = 0; for (HRegion r : regionServer.getOnlineRegionsLocalContext()) { - tempNumPutsWithoutWAL += r.numPutsWithoutWAL.get(); + tempNumMutationsWithoutWAL += r.numMutationsWithoutWAL.get(); tempDataInMemoryWithoutWAL += r.dataInMemoryWithoutWAL.get(); tempReadRequestsCount += r.readRequestsCount.get(); tempWriteRequestsCount += r.writeRequestsCount.get(); @@ -404,7 +404,7 @@ class MetricsRegionServerWrapperImpl storefileIndexSize = tempStorefileIndexSize; totalStaticIndexSize = tempTotalStaticIndexSize; totalStaticBloomSize = tempTotalStaticBloomSize; - numPutsWithoutWAL = tempNumPutsWithoutWAL; + numMutationsWithoutWAL = tempNumMutationsWithoutWAL; dataInMemoryWithoutWAL = tempDataInMemoryWithoutWAL; percentFileLocal = tempPercentFileLocal; } Modified: hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java?rev=1493308&r1=1493307&r2=1493308&view=diff ============================================================================== --- hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java (original) +++ hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java Sat Jun 15 04:23:59 2013 @@ -116,7 +116,7 @@ public class MetricsRegionServerWrapperS } @Override - public long getNumPutsWithoutWAL() { + public long getNumMutationsWithoutWAL() { return 409; } Modified: hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java?rev=1493308&r1=1493307&r2=1493308&view=diff ============================================================================== --- hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java (original) +++ hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java Sat Jun 15 04:23:59 2013 @@ -70,8 +70,8 @@ public class TestMetricsRegionServer { HELPER.assertGauge("storeFileIndexSize", 406, serverSource); HELPER.assertGauge("staticIndexSize", 407, serverSource); HELPER.assertGauge("staticBloomSize", 408, serverSource); - HELPER.assertGauge("putsWithoutWALCount", 409, serverSource); - HELPER.assertGauge("putsWithoutWALSize", 410, serverSource); + HELPER.assertGauge("mutationsWithoutWALCount", 409, serverSource); + HELPER.assertGauge("mutationsWithoutWALSize", 410, serverSource); HELPER.assertGauge("percentFilesLocal", 99, serverSource); HELPER.assertGauge("compactionQueueLength", 411, serverSource); HELPER.assertGauge("flushQueueLength", 412, serverSource); Modified: hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java?rev=1493308&r1=1493307&r2=1493308&view=diff ============================================================================== --- hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java (original) +++ hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java Sat Jun 15 04:23:59 2013 @@ -156,8 +156,8 @@ public class TestRegionServerMetrics { } @Test - public void testPutsWithoutWal() throws Exception { - byte[] tableName = Bytes.toBytes("testPutsWithoutWal"); + public void testMutationsWithoutWal() throws Exception { + byte[] tableName = Bytes.toBytes("testMutationsWithoutWal"); byte[] cf = Bytes.toBytes("d"); byte[] row = Bytes.toBytes("rk"); byte[] qualifier = Bytes.toBytes("qual"); @@ -177,9 +177,9 @@ public class TestRegionServerMetrics { t.flushCommits(); metricsRegionServer.getRegionServerWrapper().forceRecompute(); - metricsHelper.assertGauge("putsWithoutWALCount", 1, serverSource); + metricsHelper.assertGauge("mutationsWithoutWALCount", 1, serverSource); long minLength = row.length + cf.length + qualifier.length + val.length; - metricsHelper.assertGaugeGt("putsWithoutWALSize", minLength, serverSource); + metricsHelper.assertGaugeGt("mutationsWithoutWALSize", minLength, serverSource); t.close(); }