Return-Path: X-Original-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 92EA69094 for ; Wed, 21 Mar 2012 18:14:15 +0000 (UTC) Received: (qmail 72391 invoked by uid 500); 21 Mar 2012 18:14:15 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 72358 invoked by uid 500); 21 Mar 2012 18:14:15 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 72349 invoked by uid 99); 21 Mar 2012 18:14:15 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 21 Mar 2012 18:14:15 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 21 Mar 2012 18:14:13 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id BBF732388860; Wed, 21 Mar 2012 18:13:53 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1303482 - in /hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs: ./ src/main/java/org/apache/hadoop/hdfs/server/namenode/ src/test/java/org/apache/hadoop/hdfs/server/namenode/ Date: Wed, 21 Mar 2012 18:13:53 -0000 To: hdfs-commits@hadoop.apache.org From: todd@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120321181353.BBF732388860@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: todd Date: Wed Mar 21 18:13:53 2012 New Revision: 1303482 URL: http://svn.apache.org/viewvc?rev=1303482&view=rev Log: HDFS-309. FSEditLog should log progress during replay. Contributed by Sho Shimauchi. Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1303482&r1=1303481&r2=1303482&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Wed Mar 21 18:13:53 2012 @@ -153,6 +153,9 @@ Release 0.23.3 - UNRELEASED HDFS-3086. Change Datanode not to send storage list in registration. (szetszwo) + HDFS-309. FSEditLog should log progress during replay. (Sho Shimauchi + via todd) + OPTIMIZATIONS HDFS-2477. Optimize computing the diff between a block report and the Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java?rev=1303482&r1=1303481&r2=1303482&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java Wed Mar 21 18:13:53 2012 @@ -26,6 +26,8 @@ import java.io.InputStream; import java.util.Arrays; import java.util.EnumMap; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.permission.PermissionStatus; @@ -69,6 +71,8 @@ import com.google.common.base.Joiner; @InterfaceAudience.Private @InterfaceStability.Evolving public class FSEditLogLoader { + static final Log LOG = LogFactory.getLog(FSEditLogLoader.class.getName()); + static long REPLAY_TRANSACTION_LOG_INTERVAL = 1000; // 1sec private final FSNamesystem fsNamesys; public FSEditLogLoader(FSNamesystem fsNamesys) { @@ -110,6 +114,10 @@ public class FSEditLogLoader { EnumMap> opCounts = new EnumMap>(FSEditLogOpCodes.class); + if (LOG.isTraceEnabled()) { + LOG.trace("Acquiring write lock to replay edit log"); + } + fsNamesys.writeLock(); fsDir.writeLock(); @@ -117,6 +125,16 @@ public class FSEditLogLoader { Arrays.fill(recentOpcodeOffsets, -1); long txId = expectedStartingTxId - 1; + long lastTxId = in.getLastTxId(); + long numTxns = (lastTxId - expectedStartingTxId) + 1; + + long lastLogTime = now(); + + if (LOG.isDebugEnabled()) { + LOG.debug("edit log length: " + in.length() + ", start txid: " + + expectedStartingTxId + ", last txid: " + lastTxId); + } + try { try { while (true) { @@ -153,6 +171,15 @@ public class FSEditLogLoader { FSImage.LOG.error(errorMessage); throw new IOException(errorMessage, t); } + + // log progress + if (now() - lastLogTime > REPLAY_TRANSACTION_LOG_INTERVAL) { + int percent = Math.round((float) txId / numTxns * 100); + LOG.info("replaying edit log: " + txId + "/" + numTxns + + " transactions completed. (" + percent + "%)"); + lastLogTime = now(); + } + numEdits++; } } catch (IOException ex) { @@ -164,6 +191,11 @@ public class FSEditLogLoader { } finally { fsDir.writeUnlock(); fsNamesys.writeUnlock(); + + if (LOG.isTraceEnabled()) { + LOG.trace("replaying edit log finished"); + } + if (FSImage.LOG.isDebugEnabled()) { dumpOpCounts(opCounts); } @@ -174,6 +206,11 @@ public class FSEditLogLoader { @SuppressWarnings("deprecation") private void applyEditLogOp(FSEditLogOp op, FSDirectory fsDir, int logVersion) throws IOException { + + if (LOG.isTraceEnabled()) { + LOG.trace("replaying edit log: " + op); + } + switch (op.opCode) { case OP_ADD: { AddCloseOp addCloseOp = (AddCloseOp)op; Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java?rev=1303482&r1=1303481&r2=1303482&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java Wed Mar 21 18:13:53 2012 @@ -52,6 +52,7 @@ public class TestFSEditLogLoader { static { ((Log4JLogger)FSImage.LOG).getLogger().setLevel(Level.ALL); + ((Log4JLogger)FSEditLogLoader.LOG).getLogger().setLevel(Level.ALL); } private static final File TEST_DIR = new File(