hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject svn commit: r1174942 - in /hbase/trunk: ./ src/main/java/org/apache/hadoop/hbase/regionserver/ src/main/java/org/apache/hadoop/hbase/regionserver/wal/ src/main/java/org/apache/hadoop/hbase/replication/regionserver/ src/test/java/org/apache/hadoop/hbase...
Date Fri, 23 Sep 2011 18:35:17 GMT
Author: stack
Date: Fri Sep 23 18:35:17 2011
New Revision: 1174942

URL: http://svn.apache.org/viewvc?rev=1174942&view=rev
Log:
HBASE-4132 Extend the WALActionsListener API to accomodate log archival

Modified:
    hbase/trunk/CHANGES.txt
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALActionsListener.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java

Modified: hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/trunk/CHANGES.txt?rev=1174942&r1=1174941&r2=1174942&view=diff
==============================================================================
--- hbase/trunk/CHANGES.txt (original)
+++ hbase/trunk/CHANGES.txt Fri Sep 23 18:35:17 2011
@@ -1,5 +1,8 @@
 HBase Change Log
 Release 0.93.0 - Unreleased
+  IMPROVEMENT
+   HBASE-4132  Extend the WALActionsListener API to accomodate log archival
+               (dhruba borthakur)
 
 Release 0.92.0 - Unreleased
   INCOMPATIBLE CHANGES

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java?rev=1174942&r1=1174941&r2=1174942&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java Fri Sep
23 18:35:17 2011
@@ -154,7 +154,22 @@ class LogRoller extends Thread implement
   }
 
   @Override
-  public void logRolled(Path newFile) {
+  public void preLogRoll(Path oldPath, Path newPath) throws IOException {
+    // Not interested
+  }
+
+  @Override
+  public void postLogRoll(Path oldPath, Path newPath) throws IOException {
+    // Not interested
+  }
+
+  @Override
+  public void preLogArchive(Path oldPath, Path newPath) throws IOException {
+    // Not interested
+  }
+
+  @Override
+  public void postLogArchive(Path oldPath, Path newPath) throws IOException {
     // Not interested
   }
 

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java?rev=1174942&r1=1174941&r2=1174942&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java Fri Sep 23
18:35:17 2011
@@ -548,11 +548,22 @@ public class HLog implements Syncable {
       // Do all the preparation outside of the updateLock to block
       // as less as possible the incoming writes
       long currentFilenum = this.filenum;
+      Path oldPath = null;
+      if (currentFilenum > 0) {
+        oldPath = computeFilename(currentFilenum);
+      }
       this.filenum = System.currentTimeMillis();
       Path newPath = computeFilename();
       if (LOG.isDebugEnabled()) {
         LOG.debug("Enabling new writer for "+FSUtils.getPath(newPath));
       }
+
+      // Tell our listeners that a new log is about to be created
+      if (!this.listeners.isEmpty()) {
+        for (WALActionsListener i : this.listeners) {
+          i.preLogRoll(oldPath, newPath);
+        }
+      }
       HLog.Writer nextWriter = this.createWriterInstance(fs, newPath, conf);
       // Can we get at the dfsclient outputstream?  If an instance of
       // SFLW, it'll have done the necessary reflection to get at the
@@ -564,7 +575,7 @@ public class HLog implements Syncable {
       // Tell our listeners that a new log was created
       if (!this.listeners.isEmpty()) {
         for (WALActionsListener i : this.listeners) {
-          i.logRolled(newPath);
+          i.postLogRoll(oldPath, newPath);
         }
       }
 
@@ -814,9 +825,22 @@ public class HLog implements Syncable {
     LOG.info("moving old hlog file " + FSUtils.getPath(p) +
       " whose highest sequenceid is " + seqno + " to " +
       FSUtils.getPath(newPath));
+
+    // Tell our listeners that a log is going to be archived.
+    if (!this.listeners.isEmpty()) {
+      for (WALActionsListener i : this.listeners) {
+        i.preLogArchive(p, newPath);
+      }
+    }
     if (!this.fs.rename(p, newPath)) {
       throw new IOException("Unable to rename " + p + " to " + newPath);
     }
+    // Tell our listeners that a log has been archived.
+    if (!this.listeners.isEmpty()) {
+      for (WALActionsListener i : this.listeners) {
+        i.postLogArchive(p, newPath);
+      }
+    }
   }
 
   /**
@@ -851,10 +875,24 @@ public class HLog implements Syncable {
     if (!fs.exists(this.dir)) return;
     FileStatus[] files = fs.listStatus(this.dir);
     for(FileStatus file : files) {
+
       Path p = getHLogArchivePath(this.oldLogDir, file.getPath());
+      // Tell our listeners that a log is going to be archived.
+      if (!this.listeners.isEmpty()) {
+        for (WALActionsListener i : this.listeners) {
+          i.preLogArchive(file.getPath(), p);
+        }
+      }
+
       if (!fs.rename(file.getPath(),p)) {
         throw new IOException("Unable to rename " + file.getPath() + " to " + p);
       }
+      // Tell our listeners that a log was archived.
+      if (!this.listeners.isEmpty()) {
+        for (WALActionsListener i : this.listeners) {
+          i.postLogArchive(file.getPath(), p);
+        }
+      }
     }
     LOG.debug("Moved " + files.length + " log files to " +
       FSUtils.getPath(this.oldLogDir));

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALActionsListener.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALActionsListener.java?rev=1174942&r1=1174941&r2=1174942&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALActionsListener.java
(original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALActionsListener.java
Fri Sep 23 18:35:17 2011
@@ -19,6 +19,7 @@
  */
 package org.apache.hadoop.hbase.regionserver.wal;
 
+import java.io.IOException;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
@@ -28,11 +29,36 @@ import org.apache.hadoop.hbase.HTableDes
  * so make sure your implementation is fast else you'll slow hbase.
  */
 public interface WALActionsListener {
+
+  /**
+   * The WAL is going to be rolled. The oldPath can be null if this is
+   * the first log file from the regionserver.
+   * @param oldPath the path to the old hlog
+   * @param newPath the path to the new hlog
+   */
+  public void preLogRoll(Path oldPath, Path newPath) throws IOException;
+
+  /**
+   * The WAL has been rolled. The oldPath can be null if this is
+   * the first log file from the regionserver.
+   * @param oldPath the path to the old hlog
+   * @param newPath the path to the new hlog
+   */
+  public void postLogRoll(Path oldPath, Path newPath) throws IOException;
+
+  /**
+   * The WAL is going to be archived.
+   * @param oldPath the path to the old hlog
+   * @param newPath the path to the new hlog
+   */
+  public void preLogArchive(Path oldPath, Path newPath) throws IOException;
+
   /**
-   * The WAL was rolled.
-   * @param newFile the path to the new hlog
+   * The WAL has been archived.
+   * @param oldPath the path to the old hlog
+   * @param newPath the path to the new hlog
    */
-  public void logRolled(Path newFile);
+  public void postLogArchive(Path oldPath, Path newPath) throws IOException;
 
   /**
    * A request was made that the WAL be rolled.

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java?rev=1174942&r1=1174941&r2=1174942&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
(original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
Fri Sep 23 18:35:17 2011
@@ -156,8 +156,23 @@ public class Replication implements WALA
   }
 
   @Override
-  public void logRolled(Path p) {
-    getReplicationManager().logRolled(p);
+  public void preLogRoll(Path oldPath, Path newPath) throws IOException {
+    // Not interested
+  }
+
+  @Override
+  public void postLogRoll(Path oldPath, Path newPath) {
+    getReplicationManager().logRolled(newPath);
+  }
+
+  @Override
+  public void preLogArchive(Path oldPath, Path newPath) throws IOException {
+    // Not interested
+  }
+
+  @Override
+  public void postLogArchive(Path oldPath, Path newPath) throws IOException {
+    // Not interested
   }
 
   /**

Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java?rev=1174942&r1=1174941&r2=1174942&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java Fri Sep
23 18:35:17 2011
@@ -709,9 +709,23 @@ public class TestHLog  {
     }
 
     @Override
-    public void logRolled(Path newFile) {
+    public void preLogRoll(Path oldFile, Path newFile) {
+      // TODO Auto-generated method stub
+    }
+
+    @Override
+    public void postLogRoll(Path oldFile, Path newFile) {
+      // TODO Auto-generated method stub
+    }
+
+    @Override
+    public void preLogArchive(Path oldFile, Path newFile) {
+      // TODO Auto-generated method stub
+    }
+
+    @Override
+    public void postLogArchive(Path oldFile, Path newFile) {
       // TODO Auto-generated method stub
-      
     }
 
     @Override

Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java?rev=1174942&r1=1174941&r2=1174942&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java
(original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java
Fri Sep 23 18:35:17 2011
@@ -416,13 +416,22 @@ public class TestLogRolling  {
     server = TEST_UTIL.getRSForFirstRegionInTable(Bytes.toBytes(tableName));
     this.log = server.getWAL();
     final List<Path> paths = new ArrayList<Path>();
+    final List<Integer> preLogRolledCalled = new ArrayList<Integer>();
     paths.add(log.computeFilename());
     log.registerWALActionsListener(new WALActionsListener() {
       @Override
-      public void logRolled(Path newFile) {
+      public void preLogRoll(Path oldFile, Path newFile)  {
+        preLogRolledCalled.add(new Integer(1));
+      }
+      @Override
+      public void postLogRoll(Path oldFile, Path newFile) {
         paths.add(newFile);
       }
       @Override
+      public void preLogArchive(Path oldFile, Path newFile) {}
+      @Override
+      public void postLogArchive(Path oldFile, Path newFile) {}
+      @Override
       public void logRollRequested() {}
       @Override
       public void logCloseRequested() {}
@@ -478,6 +487,7 @@ public class TestLogRolling  {
 
     // force a log roll to read back and verify previously written logs
     log.rollWriter(true);
+    assertTrue(preLogRolledCalled.size() == 1);
 
     // read back the data written
     Set<String> loggedRows = new HashSet<String>();

Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java?rev=1174942&r1=1174941&r2=1174942&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java
(original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java
Fri Sep 23 18:35:17 2011
@@ -108,8 +108,10 @@ public class TestWALActionsListener {
     hlog.close();
     hlog.closeAndDelete();
 
-    assertEquals(11, observer.logRollCounter);
-    assertEquals(5, laterobserver.logRollCounter);
+    assertEquals(11, observer.preLogRollCounter);
+    assertEquals(11, observer.postLogRollCounter);
+    assertEquals(5, laterobserver.preLogRollCounter);
+    assertEquals(5, laterobserver.postLogRollCounter);
     assertEquals(2, observer.closedCount);
   }
 
@@ -118,12 +120,28 @@ public class TestWALActionsListener {
    * Just counts when methods are called
    */
   static class DummyWALActionsListener implements WALActionsListener {
-    public int logRollCounter = 0;
+    public int preLogRollCounter = 0;
+    public int postLogRollCounter = 0;
     public int closedCount = 0;
 
     @Override
-    public void logRolled(Path newFile) {
-      logRollCounter++;
+    public void preLogRoll(Path oldFile, Path newFile) {
+      preLogRollCounter++;
+    }
+
+    @Override
+    public void postLogRoll(Path oldFile, Path newFile) {
+      postLogRollCounter++;
+    }
+
+    @Override
+    public void preLogArchive(Path oldFile, Path newFile) {
+      // Not interested
+    }
+
+    @Override
+    public void postLogArchive(Path oldFile, Path newFile) {
+      // Not interested
     }
 
     @Override



Mime
View raw message