hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mbau...@apache.org
Subject svn commit: r1349308 - in /hbase/branches/0.89-fb/src/main: java/org/apache/hadoop/hbase/ java/org/apache/hadoop/hbase/regionserver/ resources/hbase-webapps/regionserver/
Date Tue, 12 Jun 2012 12:50:42 GMT
Author: mbautin
Date: Tue Jun 12 12:50:42 2012
New Revision: 1349308

URL: http://svn.apache.org/viewvc?rev=1349308&view=rev
Log:
HBASE-6194

Author: feifei

Summary: The region server currently lists all the region servers that it is hosting. It will
be useful to report when those regions were opened on this server. It will also be useful
to report what and when were the recent regions closed.

Test Plan:
create table test1, test2, test3, test4. disable these tables, then enable them. then you
will see a list of recently closed regions info and the open date of those regions.

unit test :
(1) mvn -Dtest=org.apache.hadoop.hbase.regionserver.* test
there will be an error for org.apache.hadoop.hbase.regionserver.TestFSErrorsExposed
(2) mvn -Dtest=org.apache.hadoop.hbase.io.TestHeapSize test

Reviewers: pkhemani

Reviewed By: pkhemani

CC: mbautin

Differential Revision: https://phabricator.fb.com/D486302

Task ID: 1068217

Modified:
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
    hbase/branches/0.89-fb/src/main/resources/hbase-webapps/regionserver/regionserver.jsp

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java?rev=1349308&r1=1349307&r2=1349308&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java Tue Jun
12 12:50:42 2012
@@ -627,4 +627,4 @@ public class HRegionInfo extends Version
     return isRootRegion()? KeyValue.ROOT_COMPARATOR: isMetaRegion()?
       KeyValue.META_COMPARATOR: KeyValue.COMPARATOR;
   }
-}
\ No newline at end of file
+}

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1349308&r1=1349307&r2=1349308&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
(original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
Tue Jun 12 12:50:42 2012
@@ -53,6 +53,9 @@ import java.util.concurrent.atomic.Atomi
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.Date;
+import java.util.Calendar;
+import java.text.SimpleDateFormat;
 
 import org.apache.avro.ipc.Server;
 import org.apache.commons.logging.Log;
@@ -153,6 +156,7 @@ public class HRegion implements HeapSize
   static final String SPLITDIR = "splits";
   static final String MERGEDIR = "merges";
 
+  static SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
   final AtomicBoolean closed = new AtomicBoolean(false);
   /* Closing can take some time; use the closing flag if there is stuff we don't
    * want to do while in closing state; e.g. like offer this region up to the
@@ -186,6 +190,10 @@ public class HRegion implements HeapSize
   private RequestMetrics writeRequests = null;
 
   private HRegionServer regionServer = null;
+  /**
+   * The open date of the region is stored as a long obtained by currentTimeMillis()
+   */
+  private long openDate = 0;
 
   /**
    * The directory for the table this region is part of.
@@ -376,6 +384,19 @@ public class HRegion implements HeapSize
     return m.get();
   }
 
+  public String getOpenDateAsString() {
+    Date date = new Date(openDate);
+    return formatter.format(date);
+  }
+
+  public long getopenDate() {
+    return openDate;
+  }
+
+  public void setOpenDate(long dt) {
+    openDate = dt;
+  }
+
   public static final long getWriteOps() {
     return writeOps.getAndSet(0);
   }
@@ -416,6 +437,7 @@ public class HRegion implements HeapSize
     this.regionInfo = null;
     this.threadWakeFrequency = 0L;
     this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
+    this.openDate = 0;
   }
 
   /**
@@ -2343,16 +2365,16 @@ public class HRegion implements HeapSize
    * new entries.
    */
   private long applyFamilyMapToMemstore(Map<byte[], List<KeyValue>> familyMap)
{
-	  return applyFamilyMapToMemstore(familyMap, null);
+    return applyFamilyMapToMemstore(familyMap, null);
   }
   
   private long applyFamilyMapToMemstore(Map<byte[], List<KeyValue>> familyMap,
-		  MultiVersionConsistencyControl.WriteEntry writeEntryToUse) {
+                 MultiVersionConsistencyControl.WriteEntry writeEntryToUse) {
     long start = EnvironmentEdgeManager.currentTimeMillis();
     MultiVersionConsistencyControl.WriteEntry w = null;
     long size = 0;
     try {
-	  w = (writeEntryToUse == null)? mvcc.beginMemstoreInsert(): writeEntryToUse;
+      w = (writeEntryToUse == null)? mvcc.beginMemstoreInsert(): writeEntryToUse;
 
       for (Map.Entry<byte[], List<KeyValue>> e : familyMap.entrySet()) {
         byte[] family = e.getKey();
@@ -2366,12 +2388,12 @@ public class HRegion implements HeapSize
       }
     } finally {
       if (writeEntryToUse == null) {
-	      long now = EnvironmentEdgeManager.currentTimeMillis();
-	      HRegion.memstoreInsertTime.addAndGet(now - start);
-	      start = now;
-	      mvcc.completeMemstoreInsert(w);
-	      now = EnvironmentEdgeManager.currentTimeMillis();
-	      HRegion.mvccWaitTime.addAndGet(now - start);
+        long now = EnvironmentEdgeManager.currentTimeMillis();
+        HRegion.memstoreInsertTime.addAndGet(now - start);
+        start = now;
+        mvcc.completeMemstoreInsert(w);
+        now = EnvironmentEdgeManager.currentTimeMillis();
+        HRegion.mvccWaitTime.addAndGet(now - start);
       }
       // else the calling function will take care of the mvcc completion and metrics.
     }
@@ -3755,13 +3777,13 @@ public class HRegion implements HeapSize
         flush = isFlushSize(this.incMemoryUsage(addedSize));
       } finally {
         // 8. roll mvcc forward
-	      long start = now;
-	      now = EnvironmentEdgeManager.currentTimeMillis();
-	      HRegion.memstoreInsertTime.addAndGet(now - start);
-	
-	      mvcc.completeMemstoreInsert(w);
-	      now = EnvironmentEdgeManager.currentTimeMillis();
-	      HRegion.mvccWaitTime.addAndGet(now - start);
+        long start = now;
+        now = EnvironmentEdgeManager.currentTimeMillis();
+        HRegion.memstoreInsertTime.addAndGet(now - start);
+
+        mvcc.completeMemstoreInsert(w);
+        now = EnvironmentEdgeManager.currentTimeMillis();
+        HRegion.mvccWaitTime.addAndGet(now - start);
 
         // 9. release region lock
         this.updatesLock.readLock().unlock();
@@ -3851,7 +3873,7 @@ public class HRegion implements HeapSize
     long after = EnvironmentEdgeManager.currentTimeMillis();
     String signature = SchemaMetrics.generateSchemaMetricsPrefix(
         this.getTableDesc().getNameAsString(), Bytes.toString(family));
-		HRegion.incrTimeVaryingMetric(signature + "increment_", after - before);
+    HRegion.incrTimeVaryingMetric(signature + "increment_", after - before);
 
     if (flush) {
       // Request a cache flush.  Do it outside update lock.
@@ -3876,7 +3898,7 @@ public class HRegion implements HeapSize
   }
 
   public static final long FIXED_OVERHEAD = ClassSize.align(
-      (5 * Bytes.SIZEOF_LONG) + 2 * ClassSize.ARRAY +
+      (6 * Bytes.SIZEOF_LONG) + 2 * ClassSize.ARRAY +
       (28 * ClassSize.REFERENCE) + ClassSize.OBJECT + Bytes.SIZEOF_INT);
 
   public static final long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD +

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=1349308&r1=1349307&r2=1349308&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
(original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
Tue Jun 12 12:50:42 2012
@@ -45,6 +45,9 @@ import java.util.SortedMap;
 import java.util.SortedSet;
 import java.util.TreeMap;
 import java.util.TreeSet;
+import java.util.Date;
+import java.util.Calendar;
+import java.text.SimpleDateFormat;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.LinkedBlockingQueue;
@@ -131,6 +134,7 @@ import org.apache.zookeeper.KeeperExcept
 import org.apache.zookeeper.WatchedEvent;
 import org.apache.zookeeper.Watcher;
 import org.apache.zookeeper.Watcher.Event.EventType;
+import org.apache.hadoop.hbase.util.Bytes;
 
 /**
  * HRegionServer makes a set of HRegions available to clients.  It checks in with
@@ -147,6 +151,8 @@ public class HRegionServer implements HR
   private static final String UNABLE_TO_READ_MASTER_ADDRESS_ERR_MSG =
       "Unable to read master address from ZooKeeper";
   private static final ArrayList<Put> emptyPutArray = new ArrayList<Put>();
+  private static final int DEFAULT_NUM_TRACKED_CLOSED_REGION = 3;
+  private static SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
 
   // Set when a report to the master comes back with a message asking us to
   // shutdown.  Also set by call to stop when debugging or running unit tests
@@ -183,6 +189,10 @@ public class HRegionServer implements HR
   protected final Map<Integer, HRegion> onlineRegions =
     new ConcurrentHashMap<Integer, HRegion>();
 
+  // this is a list of region info that we recently closed
+  protected final List<ClosedRegionInfo> recentlyClosedRegions =
+    new ArrayList<ClosedRegionInfo>(DEFAULT_NUM_TRACKED_CLOSED_REGION + 1);
+
   // This a list of regions that we need to re-try closing.
   protected final Map<Integer, HRegion> retryCloseRegions = Collections
       .synchronizedMap(new HashMap<Integer, HRegion>());
@@ -1045,7 +1055,49 @@ public class HRegionServer implements HR
     return;
   }
 
-  /*
+  /**
+   * Inner class that stores information for recently closed regions.
+   * including region name, closeDate, startKey, endKey.
+   */
+  public static class ClosedRegionInfo {
+    private final String regionName;
+    private final long closeDate;
+    private final String startKey;
+    private final String endKey;
+
+    ClosedRegionInfo(String name, long date, byte[] startKey, byte[] endKey) {
+      this.regionName = name;
+      this.closeDate = date;
+      this.startKey = (startKey == null) ? "" : Bytes.toStringBinary(startKey);
+      this.endKey = (endKey == null) ? "" : Bytes.toStringBinary(endKey);
+    }
+
+    public String getRegionName() {
+      return regionName;
+    }
+
+    public long getCloseDate() {
+      return closeDate;
+    }
+
+    /**
+     * the date is represented as a string in format "yyyy-MM-dd HH:mm:ss"
+     */
+    public String getCloseDateAsString() {
+      Date date = new Date(closeDate);
+      return formatter.format(date);
+    }
+
+    public String getStartKey() {
+      return startKey;
+    }
+
+    public String getEndKey() {
+      return endKey;
+    }
+  }
+
+  /**
    * Inner class that runs on a long period checking if regions need major
    * compaction.
    */
@@ -1655,6 +1707,7 @@ public class HRegionServer implements HR
 
             case MSG_REGION_CLOSE_WITHOUT_REPORT:
               // Close a region, don't reply
+
               closeRegion(e.msg.getRegionInfo(), false);
               break;
 
@@ -1783,6 +1836,7 @@ public class HRegionServer implements HR
       try {
         this.onlineRegions.put(mapKey, region);
         region.setRegionServer(this);
+        region.setOpenDate(EnvironmentEdgeManager.currentTimeMillis());
       } finally {
         this.lock.writeLock().unlock();
       }
@@ -1850,6 +1904,12 @@ public class HRegionServer implements HR
     getOutboundMsgs().add(new HMsg(HMsg.Type.MSG_REPORT_PROCESS_OPEN, hri));
   }
 
+  private void addToRecentlyClosedRegions(ClosedRegionInfo info) {
+      recentlyClosedRegions.add(0, info);
+      if (recentlyClosedRegions.size() > DEFAULT_NUM_TRACKED_CLOSED_REGION)
+        recentlyClosedRegions.remove(DEFAULT_NUM_TRACKED_CLOSED_REGION);
+  }
+
   @Override
   public void closeRegion(final HRegionInfo hri, final boolean reportWhenCompleted)
   throws IOException {
@@ -1871,6 +1931,10 @@ public class HRegionServer implements HR
         this.addToRetryCloseRegions(region);
         throw e;
       }
+      ClosedRegionInfo info =
+        new ClosedRegionInfo(hri.getRegionNameAsString(), EnvironmentEdgeManager.currentTimeMillis()
,
+                             hri.getStartKey(), hri.getEndKey());
+      addToRecentlyClosedRegions(info);
       if(reportWhenCompleted) {
         if(zkUpdater != null) {
           HMsg hmsg = new HMsg(HMsg.Type.MSG_REPORT_CLOSE, hri, null);
@@ -2071,7 +2135,7 @@ public class HRegionServer implements HR
   @Override
   public void mutateRow(byte[] regionName, RowMutations arm)
       throws IOException {
-	  mutateRow(regionName, Collections.singletonList(arm));
+          mutateRow(regionName, Collections.singletonList(arm));
   }
 
   @Override
@@ -2546,6 +2610,11 @@ public class HRegionServer implements HR
     return lock.writeLock();
   }
 
+  /** @return recent closed regions info*/
+  public List<ClosedRegionInfo> getRecentlyClosedRegionInfo() {
+    return recentlyClosedRegions;
+  }
+
   /**
    * @return Immutable list of this servers regions.
    */
@@ -2627,19 +2696,15 @@ public class HRegionServer implements HR
     return region.getLastFlushTime();
   }
 
-  /**
-   * @return The HRegionInfos from online regions sorted
-   */
-  public SortedSet<HRegionInfo> getSortedOnlineRegionInfos() {
-    SortedSet<HRegionInfo> result = new TreeSet<HRegionInfo>();
+  public Map<HRegionInfo, String> getSortedOnlineRegionInfosAndOpenDate() {
+    TreeMap<HRegionInfo, String> result = new TreeMap<HRegionInfo, String>();
     synchronized(this.onlineRegions) {
       for (HRegion r: this.onlineRegions.values()) {
-        result.add(r.getRegionInfo());
+        result.put(r.getRegionInfo(), r.getOpenDateAsString());
       }
     }
     return result;
   }
-
   /**
    * This method removes HRegion corresponding to hri from the Map of onlineRegions.
    *

Modified: hbase/branches/0.89-fb/src/main/resources/hbase-webapps/regionserver/regionserver.jsp
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/resources/hbase-webapps/regionserver/regionserver.jsp?rev=1349308&r1=1349307&r2=1349308&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/resources/hbase-webapps/regionserver/regionserver.jsp
(original)
+++ hbase/branches/0.89-fb/src/main/resources/hbase-webapps/regionserver/regionserver.jsp
Tue Jun 12 12:50:42 2012
@@ -3,6 +3,7 @@
   import="java.io.IOException"
   import="org.apache.hadoop.io.Text"
   import="org.apache.hadoop.hbase.regionserver.HRegionServer"
+  import="org.apache.hadoop.hbase.regionserver.HRegionServer.ClosedRegionInfo"
   import="org.apache.hadoop.hbase.regionserver.HRegion"
   import="org.apache.hadoop.hbase.regionserver.metrics.RegionServerMetrics"
   import="org.apache.hadoop.hbase.util.Bytes"
@@ -18,7 +19,10 @@
     e.printStackTrace();
   }
   RegionServerMetrics metrics = regionServer.getMetrics();
-  Collection<HRegionInfo> onlineRegions = regionServer.getSortedOnlineRegionInfos();
+  Map<HRegionInfo, String> onlineRegionInfoAndOpenDate = 
+        regionServer.getSortedOnlineRegionInfosAndOpenDate();
+  List<HRegionServer.ClosedRegionInfo> recentlyClosedRegionInfo = 
+        regionServer.getRecentlyClosedRegionInfo();
   int interval = regionServer.getConfiguration().getInt("hbase.regionserver.msginterval",
3000)/1000;
 
 %><?xml version="1.0" encoding="UTF-8" ?>
@@ -46,27 +50,56 @@
 </table>
 
 <h2>Online Regions</h2>
-<% if (onlineRegions != null && onlineRegions.size() > 0) { %>
+<% if (onlineRegionInfoAndOpenDate != null && onlineRegionInfoAndOpenDate.size()
> 0) { %>
 <table>
 <tr><th>Region Name</th>
 	<th>Read Requests/sec</th>
 	<th>Write Requests/sec</th>
 	<th>Start Key</th>
-	<th>End Key</th><th>Metrics</th></tr>
-<%   for (HRegionInfo r: onlineRegions) {
+	<th>End Key</th>
+  <th>Metrics</th>
+  <th>Open Time</th>
+</tr>
+<%   for (HRegionInfo r : onlineRegionInfoAndOpenDate.keySet()) {
         HServerLoad.RegionLoad load = regionServer.createRegionLoad(r.getRegionName());
  %>
+
 <tr><td><%= r.getRegionNameAsString() %></td>
 	<td><%= load.getReadRequestPerSec() %></td>
 	<td><%= load.getWriteRequestPerSec() %></td>
     <td><%= Bytes.toStringBinary(r.getStartKey()) %></td>
 	<td><%= Bytes.toStringBinary(r.getEndKey()) %></td>
     <td><%= load.toString() %></td>
+  <td><%= onlineRegionInfoAndOpenDate.get(r) %></td>
 </tr>
 <%
      }
 %>
 </table>
+<h2>Recently Closed Regions</h2>
+<% if (recentlyClosedRegionInfo != null && recentlyClosedRegionInfo.size() >
0) 
+   { %>
+<table>
+<tr><th>Region Name</th>
+  <th>Start Key</th>
+  <th>End Key</th>
+  <th>Closed Time</th>
+</tr>
+<%
+  for (HRegionServer.ClosedRegionInfo info: recentlyClosedRegionInfo) {
+%>
+<tr><td><%= info.getRegionName() %></td>
+  <td><%= info.getStartKey() %></td>
+  <td><%= info.getEndKey() %></td>
+  <td><%= info.getCloseDateAsString() %></td>
+</tr>
+<%
+  }
+%>
+</table>
+<%
+  }
+%>
 <p>Region names are made of the containing table's name, a comma,
 the start key, a comma, and a randomly generated region id.  To illustrate,
 the region named



Mime
View raw message