hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From t...@apache.org
Subject svn commit: r1303482 - in /hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs: ./ src/main/java/org/apache/hadoop/hdfs/server/namenode/ src/test/java/org/apache/hadoop/hdfs/server/namenode/
Date Wed, 21 Mar 2012 18:13:53 GMT
Author: todd
Date: Wed Mar 21 18:13:53 2012
New Revision: 1303482

URL: http://svn.apache.org/viewvc?rev=1303482&view=rev
Log:
HDFS-309. FSEditLog should log progress during replay. Contributed by Sho Shimauchi.

Modified:
    hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
    hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
    hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java

Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1303482&r1=1303481&r2=1303482&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Wed Mar
21 18:13:53 2012
@@ -153,6 +153,9 @@ Release 0.23.3 - UNRELEASED
     HDFS-3086. Change Datanode not to send storage list in registration.
     (szetszwo)
 
+    HDFS-309. FSEditLog should log progress during replay. (Sho Shimauchi
+    via todd)
+
   OPTIMIZATIONS
 
     HDFS-2477. Optimize computing the diff between a block report and the

Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java?rev=1303482&r1=1303481&r2=1303482&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
Wed Mar 21 18:13:53 2012
@@ -26,6 +26,8 @@ import java.io.InputStream;
 import java.util.Arrays;
 import java.util.EnumMap;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.permission.PermissionStatus;
@@ -69,6 +71,8 @@ import com.google.common.base.Joiner;
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
 public class FSEditLogLoader {
+  static final Log LOG = LogFactory.getLog(FSEditLogLoader.class.getName());
+  static long REPLAY_TRANSACTION_LOG_INTERVAL = 1000; // 1sec
   private final FSNamesystem fsNamesys;
 
   public FSEditLogLoader(FSNamesystem fsNamesys) {
@@ -110,6 +114,10 @@ public class FSEditLogLoader {
     EnumMap<FSEditLogOpCodes, Holder<Integer>> opCounts =
       new EnumMap<FSEditLogOpCodes, Holder<Integer>>(FSEditLogOpCodes.class);
 
+    if (LOG.isTraceEnabled()) {
+      LOG.trace("Acquiring write lock to replay edit log");
+    }
+
     fsNamesys.writeLock();
     fsDir.writeLock();
 
@@ -117,6 +125,16 @@ public class FSEditLogLoader {
     Arrays.fill(recentOpcodeOffsets, -1);
 
     long txId = expectedStartingTxId - 1;
+    long lastTxId = in.getLastTxId();
+    long numTxns = (lastTxId - expectedStartingTxId) + 1;
+
+    long lastLogTime = now();
+
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("edit log length: " + in.length() + ", start txid: "
+          + expectedStartingTxId + ", last txid: " + lastTxId);
+    }
+
     try {
       try {
         while (true) {
@@ -153,6 +171,15 @@ public class FSEditLogLoader {
             FSImage.LOG.error(errorMessage);
             throw new IOException(errorMessage, t);
           }
+
+          // log progress
+          if (now() - lastLogTime > REPLAY_TRANSACTION_LOG_INTERVAL) {
+            int percent = Math.round((float) txId / numTxns * 100);
+            LOG.info("replaying edit log: " + txId + "/" + numTxns
+                + " transactions completed. (" + percent + "%)");
+            lastLogTime = now();
+          }
+
           numEdits++;
         }
       } catch (IOException ex) {
@@ -164,6 +191,11 @@ public class FSEditLogLoader {
     } finally {
       fsDir.writeUnlock();
       fsNamesys.writeUnlock();
+
+      if (LOG.isTraceEnabled()) {
+        LOG.trace("replaying edit log finished");
+      }
+
       if (FSImage.LOG.isDebugEnabled()) {
         dumpOpCounts(opCounts);
       }
@@ -174,6 +206,11 @@ public class FSEditLogLoader {
   @SuppressWarnings("deprecation")
   private void applyEditLogOp(FSEditLogOp op, FSDirectory fsDir,
       int logVersion) throws IOException {
+
+    if (LOG.isTraceEnabled()) {
+      LOG.trace("replaying edit log: " + op);
+    }
+
     switch (op.opCode) {
     case OP_ADD: {
       AddCloseOp addCloseOp = (AddCloseOp)op;

Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java?rev=1303482&r1=1303481&r2=1303482&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java
Wed Mar 21 18:13:53 2012
@@ -52,6 +52,7 @@ public class TestFSEditLogLoader {
   
   static {
     ((Log4JLogger)FSImage.LOG).getLogger().setLevel(Level.ALL);
+    ((Log4JLogger)FSEditLogLoader.LOG).getLogger().setLevel(Level.ALL);
   }
   
   private static final File TEST_DIR = new File(



Mime
View raw message