Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id E5F57173F6 for ; Wed, 5 Nov 2014 18:14:37 +0000 (UTC) Received: (qmail 58050 invoked by uid 500); 5 Nov 2014 18:14:37 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 57978 invoked by uid 500); 5 Nov 2014 18:14:37 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 57969 invoked by uid 99); 5 Nov 2014 18:14:37 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 05 Nov 2014 18:14:37 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 5C8E790875A; Wed, 5 Nov 2014 18:14:37 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: wheat9@apache.org To: common-commits@hadoop.apache.org Message-Id: <1afa9130873f4899b0d1f2af09e7537a@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: git commit: HDFS-7357. FSNamesystem.checkFileProgress should log file path. Contributed by Tsz Wo Nicholas Sze. Date: Wed, 5 Nov 2014 18:14:37 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/trunk 203c63030 -> 18312804e HDFS-7357. FSNamesystem.checkFileProgress should log file path. Contributed by Tsz Wo Nicholas Sze. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/18312804 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/18312804 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/18312804 Branch: refs/heads/trunk Commit: 18312804e9c86c0ea6a259e288994fea6fa366ef Parents: 203c630 Author: Haohui Mai Authored: Tue Nov 4 18:03:39 2014 -0800 Committer: Haohui Mai Committed: Wed Nov 5 10:14:30 2014 -0800 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../BlockInfoUnderConstruction.java | 4 +- .../namenode/EditLogFileOutputStream.java | 6 +-- .../hdfs/server/namenode/FSEditLogLoader.java | 4 +- .../hdfs/server/namenode/FSNamesystem.java | 54 +++++++++++--------- 5 files changed, 39 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/18312804/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index be672a6..4a120e9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -338,6 +338,9 @@ Release 2.7.0 - UNRELEASED HDFS-7333. Improve logging in Storage.tryLock(). (shv) + HDFS-7357. FSNamesystem.checkFileProgress should log file path. + (Tsz Wo Nicholas Sze via wheat9) + OPTIMIZATIONS BUG FIXES http://git-wip-us.apache.org/repos/asf/hadoop/blob/18312804/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java index dd3593f..0ea7c2a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java @@ -141,7 +141,7 @@ public class BlockInfoUnderConstruction extends BlockInfo { @Override public void appendStringTo(StringBuilder sb) { - sb.append("ReplicaUnderConstruction[") + sb.append("ReplicaUC[") .append(expectedLocation) .append("|") .append(state) @@ -370,7 +370,7 @@ public class BlockInfoUnderConstruction extends BlockInfo { } private void appendUCParts(StringBuilder sb) { - sb.append("{blockUCState=").append(blockUCState) + sb.append("{UCState=").append(blockUCState) .append(", primaryNodeIndex=").append(primaryNodeIndex) .append(", replicas=["); if (replicas != null) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/18312804/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java index e9f47b9..830814c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java @@ -161,7 +161,7 @@ public class EditLogFileOutputStream extends EditLogOutputStream { fp.close(); fp = null; } finally { - IOUtils.cleanup(FSNamesystem.LOG, fc, fp); + IOUtils.cleanup(LOG, fc, fp); doubleBuf = null; fc = null; fp = null; @@ -233,8 +233,8 @@ public class EditLogFileOutputStream extends EditLogOutputStream { size += fillCapacity; total += fillCapacity; } - if(FSNamesystem.LOG.isDebugEnabled()) { - FSNamesystem.LOG.debug("Preallocated " + total + " bytes at the end of " + + if(LOG.isDebugEnabled()) { + LOG.debug("Preallocated " + total + " bytes at the end of " + "the edit log (offset " + oldSize + ")"); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/18312804/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java index 1b2b4ae..492a5ac 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java @@ -329,8 +329,8 @@ public class FSEditLogLoader { AddCloseOp addCloseOp = (AddCloseOp)op; final String path = renameReservedPathsOnUpgrade(addCloseOp.path, logVersion); - if (FSNamesystem.LOG.isDebugEnabled()) { - FSNamesystem.LOG.debug(op.opCode + ": " + path + + if (LOG.isDebugEnabled()) { + LOG.debug(op.opCode + ": " + path + " numblocks : " + addCloseOp.blocks.length + " clientHolder " + addCloseOp.clientName + " clientMachine " + addCloseOp.clientMachine); http://git-wip-us.apache.org/repos/asf/hadoop/blob/18312804/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 8c35315..52c12c0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -3151,7 +3151,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, String clientMachine = null; if(NameNode.stateChangeLog.isDebugEnabled()) { - NameNode.stateChangeLog.debug("BLOCK* NameSystem.getAdditionalBlock: " + NameNode.stateChangeLog.debug("BLOCK* getAdditionalBlock: " + src + " inodeId " + fileId + " for " + clientName); } @@ -3374,7 +3374,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, } // Check if the penultimate block is minimally replicated - if (!checkFileProgress(pendingFile, false)) { + if (!checkFileProgress(src, pendingFile, false)) { throw new NotReplicatedYetException("Not replicated yet: " + src); } return new FileState(pendingFile, src); @@ -3622,14 +3622,14 @@ public class FSNamesystem implements Namesystem, FSClusterStats, } // Check the state of the penultimate block. It should be completed // before attempting to complete the last one. - if (!checkFileProgress(pendingFile, false)) { + if (!checkFileProgress(src, pendingFile, false)) { return false; } // commit the last block and complete it if it has minimum replicas commitOrCompleteLastBlock(pendingFile, last); - if (!checkFileProgress(pendingFile, true)) { + if (!checkFileProgress(src, pendingFile, true)) { return false; } @@ -3653,8 +3653,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, throws IOException { assert hasWriteLock(); BlockInfo b = dir.addBlock(src, inodesInPath, newBlock, targets); - NameNode.stateChangeLog.info("BLOCK* allocateBlock: " + src + ". " - + getBlockPoolId() + " " + b); + NameNode.stateChangeLog.info("BLOCK* allocate " + b + " for " + src); DatanodeStorageInfo.incrementBlocksScheduled(targets); return b; } @@ -3675,30 +3674,21 @@ public class FSNamesystem implements Namesystem, FSClusterStats, * replicated. If not, return false. If checkall is true, then check * all blocks, otherwise check only penultimate block. */ - boolean checkFileProgress(INodeFile v, boolean checkall) { + private boolean checkFileProgress(String src, INodeFile v, boolean checkall) { readLock(); try { if (checkall) { - // // check all blocks of the file. - // for (BlockInfo block: v.getBlocks()) { - if (!block.isComplete()) { - LOG.info("BLOCK* checkFileProgress: " + block - + " has not reached minimal replication " - + blockManager.minReplication); + if (!isCompleteBlock(src, block, blockManager.minReplication)) { return false; } } } else { - // // check the penultimate block of this file - // BlockInfo b = v.getPenultimateBlock(); - if (b != null && !b.isComplete()) { - LOG.warn("BLOCK* checkFileProgress: " + b - + " has not reached minimal replication " - + blockManager.minReplication); + if (b != null + && !isCompleteBlock(src, b, blockManager.minReplication)) { return false; } } @@ -3708,6 +3698,19 @@ public class FSNamesystem implements Namesystem, FSClusterStats, } } + private static boolean isCompleteBlock(String src, BlockInfo b, int minRepl) { + if (!b.isComplete()) { + final BlockInfoUnderConstruction uc = (BlockInfoUnderConstruction)b; + final int numNodes = b.numNodes(); + LOG.info("BLOCK* " + b + " is not COMPLETE (ucState = " + + uc.getBlockUCState() + ", replication# = " + numNodes + + (numNodes < minRepl? " < ": " >= ") + + " minimum = " + minRepl + ") in file " + src); + return false; + } + return true; + } + //////////////////////////////////////////////////////////////// // Here's how to handle block-copy failure during client write: // -- As usual, the client's write should result in a streaming @@ -5152,9 +5155,9 @@ public class FSNamesystem implements Namesystem, FSClusterStats, if(!nameNodeHasResourcesAvailable()) { String lowResourcesMsg = "NameNode low on available disk space. "; if (!isInSafeMode()) { - FSNamesystem.LOG.warn(lowResourcesMsg + "Entering safe mode."); + LOG.warn(lowResourcesMsg + "Entering safe mode."); } else { - FSNamesystem.LOG.warn(lowResourcesMsg + "Already in safe mode."); + LOG.warn(lowResourcesMsg + "Already in safe mode."); } enterSafeMode(true); } @@ -7014,11 +7017,11 @@ public class FSNamesystem implements Namesystem, FSClusterStats, if (cacheEntry != null && cacheEntry.isSuccess()) { return; // Return previous response } - LOG.info("updatePipeline(block=" + oldBlock - + ", newGenerationStamp=" + newBlock.getGenerationStamp() + LOG.info("updatePipeline(" + oldBlock.getLocalBlock() + + ", newGS=" + newBlock.getGenerationStamp() + ", newLength=" + newBlock.getNumBytes() + ", newNodes=" + Arrays.asList(newNodes) - + ", clientName=" + clientName + + ", client=" + clientName + ")"); waitForLoadingFSImage(); writeLock(); @@ -7036,7 +7039,8 @@ public class FSNamesystem implements Namesystem, FSClusterStats, RetryCache.setState(cacheEntry, success); } getEditLog().logSync(); - LOG.info("updatePipeline(" + oldBlock + ") successfully to " + newBlock); + LOG.info("updatePipeline(" + oldBlock.getLocalBlock() + " => " + + newBlock.getLocalBlock() + ") success"); } /**