hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject svn commit: r724098 - in /hadoop/hbase/trunk: CHANGES.txt src/java/org/apache/hadoop/hbase/regionserver/HLog.java src/java/org/apache/hadoop/hbase/regionserver/LogFlusher.java src/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
Date Sun, 07 Dec 2008 07:23:25 GMT
Author: stack
Date: Sat Dec  6 23:23:25 2008
New Revision: 724098

URL: http://svn.apache.org/viewvc?rev=724098&view=rev
Log:
HBASE-1048 HLog: Found 0 logs to remove out of total 1450; oldest outstanding seqnum is 162297053
fr om region -ROOT-,,0

Modified:
    hadoop/hbase/trunk/CHANGES.txt
    hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HLog.java
    hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/LogFlusher.java
    hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/LogRoller.java

Modified: hadoop/hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/CHANGES.txt?rev=724098&r1=724097&r2=724098&view=diff
==============================================================================
--- hadoop/hbase/trunk/CHANGES.txt (original)
+++ hadoop/hbase/trunk/CHANGES.txt Sat Dec  6 23:23:25 2008
@@ -164,6 +164,8 @@
    HBASE-1030  Bit of polish on HBASE-1018
    HBASE-847   new API: HTable.getRow with numVersion specified
                (Doğacan Güney via Stack)
+   HBASE-1048  HLog: Found 0 logs to remove out of total 1450; oldest
+               outstanding seqnum is 162297053 fr om region -ROOT-,,0
 
   NEW FEATURES
    HBASE-875   Use MurmurHash instead of JenkinsHash [in bloomfilters]

Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HLog.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HLog.java?rev=724098&r1=724097&r2=724098&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HLog.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HLog.java Sat Dec  6
23:23:25 2008
@@ -137,6 +137,13 @@
   // We synchronize on updateLock to prevent updates and to prevent a log roll
   // during an update
   private final Integer updateLock = new Integer(0);
+  
+  /*
+   * If more than this many logs, force flush of oldest region to oldest edit
+   * goes to disk.  If too many and we crash, then will take forever replaying.
+   * Keep the number of logs tidy.
+   */
+  private final int maxLogs;
 
   /**
    * Create an edit log at the given <code>dir</code> location.
@@ -152,10 +159,9 @@
    * @throws IOException
    */
   public HLog(final FileSystem fs, final Path dir, final Configuration conf,
-      final LogRollListener listener) throws IOException {
-    
+    final LogRollListener listener)
+  throws IOException {
     super();
-    
     this.fs = fs;
     this.dir = dir;
     this.conf = conf;
@@ -172,6 +178,7 @@
       throw new IOException("Target HLog directory already exists: " + dir);
     }
     fs.mkdirs(dir);
+    this.maxLogs = conf.getInt("hbase.regionserver.maxlogs", 64);
     rollWriter();
   }
 
@@ -234,14 +241,17 @@
    * cacheFlushLock and then completeCacheFlush could be called which would wait
    * for the lock on this and consequently never release the cacheFlushLock
    *
+   * @return If lots of logs, flush the returned region so next time through
+   * we can clean logs. Returns null if nothing to flush.
    * @throws FailedLogCloseException
    * @throws IOException
    */
-  public void rollWriter() throws FailedLogCloseException, IOException {
+  public byte [] rollWriter() throws FailedLogCloseException, IOException {
+    byte [] regionToFlush = null;
     this.cacheFlushLock.lock();
     try {
       if (closed) {
-        return;
+        return regionToFlush;
       }
       synchronized (updateLock) {
         // Clean up current writer.
@@ -268,7 +278,7 @@
             }
             this.outputfiles.clear();
           } else {
-            cleanOldLogs();
+            regionToFlush = cleanOldLogs();
           }
         }
         this.numEntries = 0;
@@ -277,32 +287,28 @@
     } finally {
       this.cacheFlushLock.unlock();
     }
+    return regionToFlush;
   }
   
   /*
    * Clean up old commit logs.
+   * @return If lots of logs, flush the returned region so next time through
+   * we can clean logs. Returns null if nothing to flush.
    * @throws IOException
    */
-  private void cleanOldLogs() throws IOException {
-    // Get oldest edit/sequence id.  If logs are older than this id,
-    // then safe to remove.
-    Long oldestOutstandingSeqNum =
-      Collections.min(this.lastSeqWritten.values());
+  private byte [] cleanOldLogs() throws IOException {
+    byte [] regionToFlush = null;
+    Long oldestOutstandingSeqNum = getOldestOutstandingSeqNum();
     // Get the set of all log files whose final ID is older than or
     // equal to the oldest pending region operation
     TreeSet<Long> sequenceNumbers =
       new TreeSet<Long>(this.outputfiles.headMap(
         (Long.valueOf(oldestOutstandingSeqNum.longValue() + 1L))).keySet());
     // Now remove old log files (if any)
+    byte [] oldestRegion = null;
     if (LOG.isDebugEnabled()) {
       // Find region associated with oldest key -- helps debugging.
-      byte [] oldestRegion = null;
-      for (Map.Entry<byte [], Long> e: this.lastSeqWritten.entrySet()) {
-        if (e.getValue().longValue() == oldestOutstandingSeqNum.longValue()) {
-          oldestRegion = e.getKey();
-          break;
-        }
-      }
+      oldestRegion = getOldestRegion(oldestOutstandingSeqNum);
       LOG.debug("Found " + sequenceNumbers.size() + " logs to remove " +
         " out of total " + this.outputfiles.size() + "; " +
         "oldest outstanding seqnum is " + oldestOutstandingSeqNum +
@@ -313,6 +319,33 @@
         deleteLogFile(this.outputfiles.remove(seq), seq);
       }
     }
+    int countOfLogs = this.outputfiles.size() - sequenceNumbers.size();
+    if (countOfLogs > this.maxLogs) {
+      regionToFlush = oldestRegion != null?
+        oldestRegion: getOldestRegion(oldestOutstandingSeqNum);
+      LOG.info("Too many logs: logs=" + countOfLogs + ", maxlogs=" +
+        this.maxLogs + "; forcing flush of region with oldest edits: " +
+        Bytes.toString(regionToFlush));
+    }
+    return regionToFlush;
+  }
+
+  /*
+   * @return Logs older than this id are safe to remove.
+   */
+  private Long getOldestOutstandingSeqNum() {
+    return Collections.min(this.lastSeqWritten.values());
+  }
+
+  private byte [] getOldestRegion(final Long oldestOutstandingSeqNum) {
+    byte [] oldestRegion = null;
+    for (Map.Entry<byte [], Long> e: this.lastSeqWritten.entrySet()) {
+      if (e.getValue().longValue() == oldestOutstandingSeqNum.longValue()) {
+        oldestRegion = e.getKey();
+        break;
+      }
+    }
+    return oldestRegion;
   }
 
   /*

Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/LogFlusher.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/LogFlusher.java?rev=724098&r1=724097&r2=724098&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/LogFlusher.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/LogFlusher.java Sat Dec
 6 23:23:25 2008
@@ -48,7 +48,6 @@
     }
   }
 
-  @Override
   protected void chore() {
     synchronized (log) {
       HLog hlog = log.get();
@@ -57,4 +56,4 @@
       }
     }
   }
-}
+}
\ No newline at end of file

Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/LogRoller.java?rev=724098&r1=724097&r2=724098&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/LogRoller.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/LogRoller.java Sat Dec
 6 23:23:25 2008
@@ -26,9 +26,10 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.RemoteExceptionHandler;
+import org.apache.hadoop.hbase.util.Bytes;
 
 /**
- * Runs periodically to determine if the HLog should be rolled
+ * Runs periodically to determine if the HLog should be rolled.
  * 
  * NOTE: This class extends Thread rather than Chore because the sleep time
  * can be interrupted when there is something to do, rather than the Chore
@@ -61,7 +62,10 @@
       }
       rollLock.lock();          // Don't interrupt us. We're working
       try {
-        server.getLog().rollWriter();
+        byte [] regionToFlush = server.getLog().rollWriter();
+        if (regionToFlush != null) {
+          scheduleFlush(regionToFlush);
+        }
       } catch (FailedLogCloseException e) {
         LOG.fatal("Forcing server shutdown", e);
         server.abort();
@@ -79,6 +83,23 @@
     }
     LOG.info("LogRoller exiting.");
   }
+  
+  private void scheduleFlush(final byte [] region) {
+    boolean scheduled = false;
+    HRegion r = this.server.getOnlineRegion(region);
+    FlushRequester requester = null;
+    if (r != null) {
+      requester = this.server.getFlushRequester();
+      if (requester != null) {
+        requester.request(r);
+        scheduled = true;
+      }
+    }
+    if (!scheduled) {
+    LOG.warn("Failed to schedule flush of " +
+      Bytes.toString(region) + "r=" + r + ", requester=" + requester);
+    }
+  }
 
   public void logRollRequested() {
     synchronized (rollLog) {



Mime
View raw message