hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From te...@apache.org
Subject svn commit: r1331683 - in /hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver: HRegion.java RegionServerAccounting.java handler/OpenRegionHandler.java
Date Sat, 28 Apr 2012 04:17:39 GMT
Author: tedyu
Date: Sat Apr 28 04:17:39 2012
New Revision: 1331683

URL: http://svn.apache.org/viewvc?rev=1331683&view=rev
Log:
HBASE-5611 Replayed edits from regions that failed to open during recovery aren't removed
from the global MemStore size (Jieshan)

Modified:
    hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
    hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java
    hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1331683&r1=1331682&r2=1331683&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java Sat
Apr 28 04:17:39 2012
@@ -299,6 +299,7 @@ public class HRegion implements HeapSize
   final long timestampSlop;
   private volatile long lastFlushTime;
   final RegionServerServices rsServices;
+  private RegionServerAccounting rsAccounting;
   private List<Pair<Long, Long>> recentFlushes = new ArrayList<Pair<Long,Long>>();
   private long blockingMemStoreSize;
   final long threadWakeFrequency;
@@ -403,9 +404,10 @@ public class HRegion implements HeapSize
         "hbase.hregion.keyvalue.timestamp.slop.millisecs",
         HConstants.LATEST_TIMESTAMP);
 
-    // don't initialize coprocessors if not running within a regionserver
-    // TODO: revisit if coprocessors should load in other cases
     if (rsServices != null) {
+      this.rsAccounting = this.rsServices.getRegionServerAccounting();
+      // don't initialize coprocessors if not running within a regionserver
+      // TODO: revisit if coprocessors should load in other cases
       this.coprocessorHost = new RegionCoprocessorHost(this, rsServices, conf);
     }
     if (LOG.isDebugEnabled()) {
@@ -662,14 +664,9 @@ public class HRegion implements HeapSize
    * @return the size of memstore in this region
    */
   public long addAndGetGlobalMemstoreSize(long memStoreSize) {
-    if (this.rsServices != null) {
-      RegionServerAccounting rsAccounting =
-        this.rsServices.getRegionServerAccounting();
-
-      if (rsAccounting != null) {
-        rsAccounting.addAndGetGlobalMemstoreSize(memStoreSize);
-      }
-    }
+    if (this.rsAccounting != null) {
+      rsAccounting.addAndGetGlobalMemstoreSize(memStoreSize);
+    }  
     return this.memstoreSize.getAndAdd(memStoreSize);
   }
 
@@ -2653,6 +2650,11 @@ public class HRegion implements HeapSize
           throw e;
         }
       }
+      // The edits size added into rsAccounting during this replaying will not
+      // be required any more. So just clear it.
+      if (this.rsAccounting != null) {
+        this.rsAccounting.clearRegionReplayEditsSize(this.regionInfo.getRegionName());
+      }
     }
     if (seqid > minSeqId) {
       // Then we added some edits to memory. Flush and cleanup split edit files.
@@ -2834,7 +2836,11 @@ public class HRegion implements HeapSize
    * @return True if we should flush.
    */
   protected boolean restoreEdit(final Store s, final KeyValue kv) {
-    return isFlushSize(this.addAndGetGlobalMemstoreSize(s.add(kv)));
+    long kvSize = s.add(kv);
+    if (this.rsAccounting != null) {
+      rsAccounting.addAndGetRegionReplayEditsSize(this.regionInfo.getRegionName(), kvSize);
+    }
+    return isFlushSize(this.addAndGetGlobalMemstoreSize(kvSize));
   }
 
   /*
@@ -4611,7 +4617,7 @@ public class HRegion implements HeapSize
   public static final long FIXED_OVERHEAD = ClassSize.align(
       ClassSize.OBJECT +
       ClassSize.ARRAY +
-      30 * ClassSize.REFERENCE + Bytes.SIZEOF_INT +
+      31 * ClassSize.REFERENCE + Bytes.SIZEOF_INT +
       (6 * Bytes.SIZEOF_LONG) +
       Bytes.SIZEOF_BOOLEAN);
 

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java?rev=1331683&r1=1331682&r2=1331683&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java
(original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java
Sat Apr 28 04:17:39 2012
@@ -19,8 +19,12 @@
  */
 package org.apache.hadoop.hbase.regionserver;
 
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ConcurrentSkipListMap;
 import java.util.concurrent.atomic.AtomicLong;
 
+import org.apache.hadoop.hbase.util.Bytes;
+
 /**
  * RegionServerAccounting keeps record of some basic real time information about
  * the Region Server. Currently, it only keeps record the global memstore size. 
@@ -29,6 +33,11 @@ public class RegionServerAccounting {
 
   private final AtomicLong atomicGlobalMemstoreSize = new AtomicLong(0);
   
+  // Store the edits size during replaying HLog. Use this to roll back the  
+  // global memstore size once a region opening failed.
+  private final ConcurrentMap<byte[], AtomicLong> replayEditsPerRegion = 
+    new ConcurrentSkipListMap<byte[], AtomicLong>(Bytes.BYTES_COMPARATOR);
+  
   /**
    * @return the global Memstore size in the RegionServer
    */
@@ -44,5 +53,46 @@ public class RegionServerAccounting {
   public long addAndGetGlobalMemstoreSize(long memStoreSize) {
     return atomicGlobalMemstoreSize.addAndGet(memStoreSize);
   }
- 
+
+  /***
+   * Add memStoreSize to replayEditsPerRegion.
+   * 
+   * @param regionName region name.
+   * @param memStoreSize the Memstore size will be added to replayEditsPerRegion.
+   * @return the replay edits size for region hri.
+   */
+  public long addAndGetRegionReplayEditsSize(byte[] regionName, long memStoreSize) {
+    AtomicLong replayEdistsSize = replayEditsPerRegion.get(regionName);
+    if (replayEdistsSize == null) {
+      replayEdistsSize = new AtomicLong(0);
+      replayEditsPerRegion.put(regionName, replayEdistsSize);
+    }
+    return replayEdistsSize.addAndGet(memStoreSize);
+  }
+
+  /**
+   * Roll back the global MemStore size for a specified region when this region
+   * can't be opened.
+   * 
+   * @param regionName the region which could not open.
+   * @return the global Memstore size in the RegionServer
+   */
+  public long rollbackRegionReplayEditsSize(byte[] regionName) {
+    AtomicLong replayEditsSize = replayEditsPerRegion.get(regionName);
+    long editsSizeLong = 0L;
+    if (replayEditsSize != null) {
+      editsSizeLong = -replayEditsSize.get();
+      clearRegionReplayEditsSize(regionName);
+    }
+    return addAndGetGlobalMemstoreSize(editsSizeLong);
+  }
+
+  /**
+   * Clear a region from replayEditsPerRegion.
+   * 
+   * @param regionName region name.
+   */
+  public void clearRegionReplayEditsSize(byte[] regionName) {
+    replayEditsPerRegion.remove(regionName);
+  }
 }

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java?rev=1331683&r1=1331682&r2=1331683&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
(original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
Sat Apr 28 04:17:39 2012
@@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.HTableDes
 import org.apache.hadoop.hbase.Server;
 import org.apache.hadoop.hbase.executor.EventHandler;
 import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.RegionServerAccounting;
 import org.apache.hadoop.hbase.regionserver.RegionServerServices;
 import org.apache.hadoop.hbase.util.CancelableProgressable;
 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
@@ -343,8 +344,17 @@ public class OpenRegionHandler extends E
       // We failed open. Our caller will see the 'null' return value
       // and transition the node back to FAILED_OPEN. If that fails,
       // we rely on the Timeout Monitor in the master to reassign.
-      LOG.error("Failed open of region=" +
-        this.regionInfo.getRegionNameAsString(), t);
+      LOG.error(
+          "Failed open of region=" + this.regionInfo.getRegionNameAsString()
+              + ", starting to roll back the global memstore size.", t);
+      // Decrease the global memstore size.
+      if (this.rsServices != null) {
+        RegionServerAccounting rsAccounting =
+          this.rsServices.getRegionServerAccounting();
+        if (rsAccounting != null) {
+          rsAccounting.rollbackRegionReplayEditsSize(this.regionInfo.getRegionName());
+        }
+      }
     }
     return region;
   }



Mime
View raw message