hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject svn commit: r1205723 - in /hbase/branches/0.92/src: main/java/org/apache/hadoop/hbase/regionserver/HRegion.java main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java test/java/org/apache/hadoop/hbase/TestGlobalMemStoreSize.java
Date Thu, 24 Nov 2011 04:32:08 GMT
Author: stack
Date: Thu Nov 24 04:32:07 2011
New Revision: 1205723

URL: http://svn.apache.org/viewvc?rev=1205723&view=rev
Log:
HBASE-4853 HBASE-4789 does overzealous pruning of seqids

Modified:
    hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
    hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
    hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/TestGlobalMemStoreSize.java

Modified: hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1205723&r1=1205722&r2=1205723&view=diff
==============================================================================
--- hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java Thu
Nov 24 04:32:07 2011
@@ -1165,8 +1165,8 @@ public class HRegion implements HeapSize
     // rows then)
     status.setStatus("Obtaining lock to block concurrent updates");
     this.updatesLock.writeLock().lock();
+    long flushsize = this.memstoreSize.get();
     status.setStatus("Preparing to flush by snapshotting stores");
-    long currentMemStoreSize = 0;
     List<StoreFlusher> storeFlushers = new ArrayList<StoreFlusher>(stores.size());
     try {
       sequenceId = (wal == null)? myseqid:
@@ -1184,7 +1184,8 @@ public class HRegion implements HeapSize
     } finally {
       this.updatesLock.writeLock().unlock();
     }
-    String s = "Finished snapshotting " + this + ", commencing wait for rwcc";
+    String s = "Finished snapshotting " + this +
+      ", commencing wait for mvcc, flushsize=" + flushsize;
     status.setStatus(s);
     LOG.debug(s);
 
@@ -1212,8 +1213,7 @@ public class HRegion implements HeapSize
       storeFlushers.clear();
 
       // Set down the memstore size by amount of flush.
-      currentMemStoreSize =
-        this.addAndGetGlobalMemstoreSize(-this.memstoreSize.get());
+      this.addAndGetGlobalMemstoreSize(-flushsize);
     } catch (Throwable t) {
       // An exception here means that the snapshot was not persisted.
       // The hlog needs to be replayed so its content is restored to memstore.
@@ -1251,14 +1251,17 @@ public class HRegion implements HeapSize
     }
 
     long time = EnvironmentEdgeManager.currentTimeMillis() - startTime;
+    long memstoresize = this.memstoreSize.get();
     String msg = "Finished memstore flush of ~" +
-        StringUtils.humanReadableInt(currentMemStoreSize) + " for region " +
-        this + " in " + time + "ms, sequenceid=" + sequenceId +
-        ", compaction requested=" + compactionRequested +
-        ((wal == null)? "; wal=null": "");
+      StringUtils.humanReadableInt(flushsize) + "/" + flushsize +
+      ", currentsize=" +
+      StringUtils.humanReadableInt(memstoresize) + "/" + memstoresize +
+      " for region " + this + " in " + time + "ms, sequenceid=" + sequenceId +
+      ", compaction requested=" + compactionRequested +
+      ((wal == null)? "; wal=null": "");
     LOG.info(msg);
     status.setStatus(msg);
-    this.recentFlushes.add(new Pair<Long,Long>(time/1000,currentMemStoreSize));
+    this.recentFlushes.add(new Pair<Long,Long>(time/1000, flushsize));
 
     return compactionRequested;
   }

Modified: hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java?rev=1205723&r1=1205722&r2=1205723&view=diff
==============================================================================
--- hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java (original)
+++ hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java Thu
Nov 24 04:32:07 2011
@@ -1359,11 +1359,6 @@ public class HLog implements Syncable {
       // Cleaning up of lastSeqWritten is in the finally clause because we
       // don't want to confuse getOldestOutstandingSeqNum()
       this.lastSeqWritten.remove(getSnapshotName(encodedRegionName));
-      Long l = this.lastSeqWritten.remove(encodedRegionName);
-      if (l != null) {
-        LOG.warn("Why is there a raw encodedRegionName in lastSeqWritten? name=" +
-          Bytes.toString(encodedRegionName) + ", seqid=" + l);
-       }
       this.cacheFlushLock.unlock();
     }
   }

Modified: hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/TestGlobalMemStoreSize.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/TestGlobalMemStoreSize.java?rev=1205723&r1=1205722&r2=1205723&view=diff
==============================================================================
--- hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/TestGlobalMemStoreSize.java
(original)
+++ hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/TestGlobalMemStoreSize.java
Thu Nov 24 04:32:07 2011
@@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.client.HT
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.util.JVMClusterUtil;
 import org.junit.Test;
 
@@ -67,7 +68,7 @@ public class TestGlobalMemStoreSize {
     cluster = TEST_UTIL.getHBaseCluster();
     LOG.info("Waiting for active/ready master");
     cluster.waitForActiveAndReadyMaster();
-    
+
     // Create a table with regions
     byte [] table = Bytes.toBytes("TestGlobalMemStoreSize");
     byte [] family = Bytes.toBytes("family");
@@ -92,20 +93,56 @@ public class TestGlobalMemStoreSize {
     // check the global memstore size after flush
     int i = 0;
     for (HRegionServer server : getOnlineRegionServers()) {
-      LOG.info("Starting flushes on " + server.getServerName() + ", size=" + server.getRegionServerAccounting().getGlobalMemstoreSize());
-      for(HRegionInfo regionInfo : server.getOnlineRegions()) {
-        HRegion region= 
-          server.getFromOnlineRegions(regionInfo.getEncodedName());
-        LOG.info("Flush " + region.toString() + " on " + server.getServerName() + ", " +
-          region.flushcache() + ", size=" + server.getRegionServerAccounting().getGlobalMemstoreSize());;
+      LOG.info("Starting flushes on " + server.getServerName() +
+        ", size=" + server.getRegionServerAccounting().getGlobalMemstoreSize());
+
+      for (HRegionInfo regionInfo : server.getOnlineRegions()) {
+        HRegion r = server.getFromOnlineRegions(regionInfo.getEncodedName());
+        flush(r, server);
       }
-      assertEquals(server.getRegionServerAccounting().getGlobalMemstoreSize(),
-          0);
+      LOG.info("Post flush on " + server.getServerName());
+      long now = System.currentTimeMillis();
+      long timeout = now + 1000;
+      while(server.getRegionServerAccounting().getGlobalMemstoreSize() != 0 &&
+          timeout < System.currentTimeMillis()) {
+        Threads.sleep(10);
+      }
+      long size = server.getRegionServerAccounting().getGlobalMemstoreSize();
+      if (size > 0) {
+        // If size > 0, see if its because the meta region got edits while
+        // our test was running....
+        for (HRegionInfo regionInfo : server.getOnlineRegions()) {
+          HRegion r = server.getFromOnlineRegions(regionInfo.getEncodedName());
+          long l = r.getMemstoreSize().longValue();
+          if (l > 0) {
+            // Only meta could have edits at this stage.  Give it another flush
+            // clear them.
+            assertTrue(regionInfo.isMetaRegion());
+            LOG.info(r.toString() + " " + l + ", reflushing");
+            r.flushcache();
+          }
+        }
+      }
+      size = server.getRegionServerAccounting().getGlobalMemstoreSize();
+      assertEquals("Server=" + server.getServerName() + ", i=" + i++, 0, size);
     }
 
     TEST_UTIL.shutdownMiniCluster();
   }
   
+  /**
+   * Flush and log stats on flush
+   * @param r
+   * @param server
+   * @throws IOException
+   */
+  private void flush(final HRegion r, final HRegionServer server)
+  throws IOException {
+    LOG.info("Flush " + r.toString() + " on " + server.getServerName() +
+      ", " +  r.flushcache() + ", size=" +
+      server.getRegionServerAccounting().getGlobalMemstoreSize());
+  }
+
   /** figure out how many regions are currently being served. */
   private int getRegionCount() throws IOException {
     int total = 0;
@@ -138,4 +175,3 @@ public class TestGlobalMemStoreSize {
     }
   }
 }
-



Mime
View raw message