hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject svn commit: r1493308 - in /hbase/branches/0.95: hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/ hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/ hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/...
Date Sat, 15 Jun 2013 04:23:59 GMT
Author: stack
Date: Sat Jun 15 04:23:59 2013
New Revision: 1493308

URL: http://svn.apache.org/r1493308
Log:
HBASE-8689 Cover all mutations rather than only Put while reporting for mutations not writing
to WAL

Modified:
    hbase/branches/0.95/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
    hbase/branches/0.95/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
    hbase/branches/0.95/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
    hbase/branches/0.95/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
    hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
    hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
    hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
    hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
    hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java

Modified: hbase/branches/0.95/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java?rev=1493308&r1=1493307&r2=1493308&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
(original)
+++ hbase/branches/0.95/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
Sat Jun 15 04:23:59 2013
@@ -145,10 +145,10 @@ public interface MetricsRegionServerSour
   static final String STATIC_BLOOM_SIZE = "staticBloomSize";
   static final String STATIC_BLOOM_SIZE_DESC =
       "Uncompressed size of the static bloom filters.";
-  static final String NUMBER_OF_PUTS_WITHOUT_WAL = "putsWithoutWALCount";
-  static final String NUMBER_OF_PUTS_WITHOUT_WAL_DESC =
+  static final String NUMBER_OF_MUTATIONS_WITHOUT_WAL = "mutationsWithoutWALCount";
+  static final String NUMBER_OF_MUTATIONS_WITHOUT_WAL_DESC =
       "Number of mutations that have been sent by clients with the write ahead logging turned
off.";
-  static final String DATA_SIZE_WITHOUT_WAL = "putsWithoutWALSize";
+  static final String DATA_SIZE_WITHOUT_WAL = "mutationsWithoutWALSize";
   static final String DATA_SIZE_WITHOUT_WAL_DESC =
       "Size of data that has been sent by clients with the write ahead logging turned off.";
   static final String PERCENT_FILES_LOCAL = "percentFilesLocal";

Modified: hbase/branches/0.95/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java?rev=1493308&r1=1493307&r2=1493308&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
(original)
+++ hbase/branches/0.95/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
Sat Jun 15 04:23:59 2013
@@ -130,7 +130,7 @@ public interface MetricsRegionServerWrap
   /**
    * Number of mutations received with WAL explicitly turned off.
    */
-  long getNumPutsWithoutWAL();
+  long getNumMutationsWithoutWAL();
 
   /**
    * Ammount of data in the memstore but not in the WAL because mutations explicitly had
their

Modified: hbase/branches/0.95/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java?rev=1493308&r1=1493307&r2=1493308&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
(original)
+++ hbase/branches/0.95/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
Sat Jun 15 04:23:59 2013
@@ -164,9 +164,9 @@ public class MetricsRegionServerSourceIm
           .addGauge(STOREFILE_INDEX_SIZE, STOREFILE_INDEX_SIZE_DESC, rsWrap.getStoreFileIndexSize())
           .addGauge(STATIC_INDEX_SIZE, STATIC_INDEX_SIZE_DESC, rsWrap.getTotalStaticIndexSize())
           .addGauge(STATIC_BLOOM_SIZE, STATIC_BLOOM_SIZE_DESC, rsWrap.getTotalStaticBloomSize())
-          .addGauge(NUMBER_OF_PUTS_WITHOUT_WAL,
-              NUMBER_OF_PUTS_WITHOUT_WAL_DESC,
-              rsWrap.getNumPutsWithoutWAL())
+          .addGauge(NUMBER_OF_MUTATIONS_WITHOUT_WAL,
+              NUMBER_OF_MUTATIONS_WITHOUT_WAL_DESC,
+              rsWrap.getNumMutationsWithoutWAL())
           .addGauge(DATA_SIZE_WITHOUT_WAL,
               DATA_SIZE_WITHOUT_WAL_DESC,
               rsWrap.getDataInMemoryWithoutWAL())

Modified: hbase/branches/0.95/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java?rev=1493308&r1=1493307&r2=1493308&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
(original)
+++ hbase/branches/0.95/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
Sat Jun 15 04:23:59 2013
@@ -173,9 +173,10 @@ public class MetricsRegionServerSourceIm
           .addGauge(Interns.info(STATIC_INDEX_SIZE, STATIC_INDEX_SIZE_DESC),
               rsWrap.getTotalStaticIndexSize())
           .addGauge(Interns.info(STATIC_BLOOM_SIZE, STATIC_BLOOM_SIZE_DESC),
-              rsWrap.getTotalStaticBloomSize())
-          .addGauge(Interns.info(NUMBER_OF_PUTS_WITHOUT_WAL, NUMBER_OF_PUTS_WITHOUT_WAL_DESC),
-              rsWrap.getNumPutsWithoutWAL())
+            rsWrap.getTotalStaticBloomSize())
+          .addGauge(
+            Interns.info(NUMBER_OF_MUTATIONS_WITHOUT_WAL, NUMBER_OF_MUTATIONS_WITHOUT_WAL_DESC),
+              rsWrap.getNumMutationsWithoutWAL())
           .addGauge(Interns.info(DATA_SIZE_WITHOUT_WAL, DATA_SIZE_WITHOUT_WAL_DESC),
               rsWrap.getDataInMemoryWithoutWAL())
           .addGauge(Interns.info(PERCENT_FILES_LOCAL, PERCENT_FILES_LOCAL_DESC),

Modified: hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1493308&r1=1493307&r2=1493308&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
(original)
+++ hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
Sat Jun 15 04:23:59 2013
@@ -232,7 +232,7 @@ public class HRegion implements HeapSize
   public final AtomicLong memstoreSize = new AtomicLong(0);
 
   // Debug possible data loss due to WAL off
-  final Counter numPutsWithoutWAL = new Counter();
+  final Counter numMutationsWithoutWAL = new Counter();
   final Counter dataInMemoryWithoutWAL = new Counter();
 
   // Debug why CAS operations are taking a while.
@@ -502,7 +502,6 @@ public class HRegion implements HeapSize
     // When hbase.regionserver.optionallogflushinterval <= 0 , deferred log sync is disabled.
     this.deferredLogSyncDisabled = conf.getLong("hbase.regionserver.optionallogflushinterval",
         1 * 1000) <= 0;
-
     if (rsServices != null) {
       this.rsAccounting = this.rsServices.getRegionServerAccounting();
       // don't initialize coprocessors if not running within a regionserver
@@ -1367,8 +1366,8 @@ public class HRegion implements HeapSize
         status.setStatus("Running coprocessor pre-flush hooks");
         coprocessorHost.preFlush();
       }
-      if (numPutsWithoutWAL.get() > 0) {
-        numPutsWithoutWAL.set(0);
+      if (numMutationsWithoutWAL.get() > 0) {
+        numMutationsWithoutWAL.set(0);
         dataInMemoryWithoutWAL.set(0);
       }
       synchronized (writestate) {
@@ -2224,9 +2223,7 @@ public class HRegion implements HeapSize
           durability = tmpDur;
         }
         if (tmpDur == Durability.SKIP_WAL) {
-          if (m instanceof Put) {
-            recordPutWithoutWal(m.getFamilyMap());
-          }
+          recordMutationWithoutWal(m.getFamilyMap());
           continue;
         }
 
@@ -4869,10 +4866,11 @@ public class HRegion implements HeapSize
           // Using default cluster id, as this can only happen in the orginating
           // cluster. A slave cluster receives the final value (not the delta)
           // as a Put.
-          txid = this.log.appendNoSync(this.getRegionInfo(),
-              this.htableDescriptor.getName(), walEdits,
-              HConstants.DEFAULT_CLUSTER_ID, EnvironmentEdgeManager.currentTimeMillis(),
-              this.htableDescriptor);
+          txid = this.log.appendNoSync(this.getRegionInfo(), this.htableDescriptor.getName(),
+            walEdits, HConstants.DEFAULT_CLUSTER_ID, EnvironmentEdgeManager.currentTimeMillis(),
+            this.htableDescriptor);
+        } else {
+          recordMutationWithoutWal(append.getFamilyMap());
         }
 
         //Actually write to Memstore now
@@ -5017,8 +5015,9 @@ public class HRegion implements HeapSize
           txid = this.log.appendNoSync(this.getRegionInfo(), this.htableDescriptor.getName(),
               walEdits, HConstants.DEFAULT_CLUSTER_ID, EnvironmentEdgeManager.currentTimeMillis(),
               this.htableDescriptor);
+        } else {
+          recordMutationWithoutWal(increment.getFamilyMap());
         }
-
         //Actually write to Memstore now
         for (Map.Entry<Store, List<KeyValue>> entry : tempMemstore.entrySet())
{
           Store store = entry.getKey();
@@ -5445,22 +5444,22 @@ public class HRegion implements HeapSize
    * Update counters for numer of puts without wal and the size of possible data loss.
    * These information are exposed by the region server metrics.
    */
-  private void recordPutWithoutWal(final Map<byte [], List<? extends Cell>> familyMap)
{
-    numPutsWithoutWAL.increment();
-    if (numPutsWithoutWAL.get() <= 1) {
+  private void recordMutationWithoutWal(final Map<byte [], List<? extends Cell>>
familyMap) {
+    numMutationsWithoutWAL.increment();
+    if (numMutationsWithoutWAL.get() <= 1) {
       LOG.info("writing data to region " + this +
                " with WAL disabled. Data may be lost in the event of a crash.");
     }
 
-    long putSize = 0;
+    long mutationSize = 0;
     for (List<? extends Cell> cells: familyMap.values()) {
       for (Cell cell : cells) {
         KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
-        putSize += kv.getKeyLength() + kv.getValueLength();
+        mutationSize += kv.getKeyLength() + kv.getValueLength();
       }
     }
 
-    dataInMemoryWithoutWAL.add(putSize);
+    dataInMemoryWithoutWAL.add(mutationSize);
   }
 
   private void lock(final Lock lock)

Modified: hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java?rev=1493308&r1=1493307&r2=1493308&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
(original)
+++ hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
Sat Jun 15 04:23:59 2013
@@ -62,7 +62,7 @@ class MetricsRegionServerWrapperImpl
   private volatile long storefileIndexSize = 0;
   private volatile long totalStaticIndexSize = 0;
   private volatile long totalStaticBloomSize = 0;
-  private volatile long numPutsWithoutWAL = 0;
+  private volatile long numMutationsWithoutWAL = 0;
   private volatile long dataInMemoryWithoutWAL = 0;
   private volatile int percentFileLocal = 0;
 
@@ -292,8 +292,8 @@ class MetricsRegionServerWrapperImpl
   }
 
   @Override
-  public long getNumPutsWithoutWAL() {
-    return numPutsWithoutWAL;
+  public long getNumMutationsWithoutWAL() {
+    return numMutationsWithoutWAL;
   }
 
   @Override
@@ -344,13 +344,13 @@ class MetricsRegionServerWrapperImpl
       long tempStorefileIndexSize = 0;
       long tempTotalStaticIndexSize = 0;
       long tempTotalStaticBloomSize = 0;
-      long tempNumPutsWithoutWAL = 0;
+      long tempNumMutationsWithoutWAL = 0;
       long tempDataInMemoryWithoutWAL = 0;
       int tempPercentFileLocal = 0;
 
 
       for (HRegion r : regionServer.getOnlineRegionsLocalContext()) {
-        tempNumPutsWithoutWAL += r.numPutsWithoutWAL.get();
+        tempNumMutationsWithoutWAL += r.numMutationsWithoutWAL.get();
         tempDataInMemoryWithoutWAL += r.dataInMemoryWithoutWAL.get();
         tempReadRequestsCount += r.readRequestsCount.get();
         tempWriteRequestsCount += r.writeRequestsCount.get();
@@ -404,7 +404,7 @@ class MetricsRegionServerWrapperImpl
       storefileIndexSize = tempStorefileIndexSize;
       totalStaticIndexSize = tempTotalStaticIndexSize;
       totalStaticBloomSize = tempTotalStaticBloomSize;
-      numPutsWithoutWAL = tempNumPutsWithoutWAL;
+      numMutationsWithoutWAL = tempNumMutationsWithoutWAL;
       dataInMemoryWithoutWAL = tempDataInMemoryWithoutWAL;
       percentFileLocal = tempPercentFileLocal;
     }

Modified: hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java?rev=1493308&r1=1493307&r2=1493308&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
(original)
+++ hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
Sat Jun 15 04:23:59 2013
@@ -116,7 +116,7 @@ public class MetricsRegionServerWrapperS
   }
 
   @Override
-  public long getNumPutsWithoutWAL() {
+  public long getNumMutationsWithoutWAL() {
     return 409;
   }
 

Modified: hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java?rev=1493308&r1=1493307&r2=1493308&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
(original)
+++ hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
Sat Jun 15 04:23:59 2013
@@ -70,8 +70,8 @@ public class TestMetricsRegionServer {
     HELPER.assertGauge("storeFileIndexSize", 406, serverSource);
     HELPER.assertGauge("staticIndexSize", 407, serverSource);
     HELPER.assertGauge("staticBloomSize", 408, serverSource);
-    HELPER.assertGauge("putsWithoutWALCount", 409, serverSource);
-    HELPER.assertGauge("putsWithoutWALSize", 410, serverSource);
+    HELPER.assertGauge("mutationsWithoutWALCount", 409, serverSource);
+    HELPER.assertGauge("mutationsWithoutWALSize", 410, serverSource);
     HELPER.assertGauge("percentFilesLocal", 99, serverSource);
     HELPER.assertGauge("compactionQueueLength", 411, serverSource);
     HELPER.assertGauge("flushQueueLength", 412, serverSource);

Modified: hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java?rev=1493308&r1=1493307&r2=1493308&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
(original)
+++ hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
Sat Jun 15 04:23:59 2013
@@ -156,8 +156,8 @@ public class TestRegionServerMetrics {
   }
 
   @Test
-  public void testPutsWithoutWal() throws Exception {
-    byte[] tableName = Bytes.toBytes("testPutsWithoutWal");
+  public void testMutationsWithoutWal() throws Exception {
+    byte[] tableName = Bytes.toBytes("testMutationsWithoutWal");
     byte[] cf = Bytes.toBytes("d");
     byte[] row = Bytes.toBytes("rk");
     byte[] qualifier = Bytes.toBytes("qual");
@@ -177,9 +177,9 @@ public class TestRegionServerMetrics {
     t.flushCommits();
 
     metricsRegionServer.getRegionServerWrapper().forceRecompute();
-    metricsHelper.assertGauge("putsWithoutWALCount", 1, serverSource);
+    metricsHelper.assertGauge("mutationsWithoutWALCount", 1, serverSource);
     long minLength = row.length + cf.length + qualifier.length + val.length;
-    metricsHelper.assertGaugeGt("putsWithoutWALSize", minLength, serverSource);
+    metricsHelper.assertGaugeGt("mutationsWithoutWALSize", minLength, serverSource);
 
     t.close();
   }



Mime
View raw message