Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 47BE01868F for ; Mon, 14 Mar 2016 17:47:02 +0000 (UTC) Received: (qmail 33598 invoked by uid 500); 14 Mar 2016 17:47:01 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 33541 invoked by uid 500); 14 Mar 2016 17:47:01 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 33532 invoked by uid 99); 14 Mar 2016 17:47:01 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 14 Mar 2016 17:47:01 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id A124CDFAEC; Mon, 14 Mar 2016 17:47:01 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: cnauroth@apache.org To: common-commits@hadoop.apache.org Date: Mon, 14 Mar 2016 17:47:01 -0000 Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: [1/3] hadoop git commit: HDFS-9941. Do not log StandbyException on NN, other minor logging fixes. Contributed by Arpit Agarwal. Repository: hadoop Updated Branches: refs/heads/branch-2 3d5ac829d -> dd462bc32 refs/heads/branch-2.8 4f5b496b8 -> e6c6f4dda refs/heads/trunk f291d82cd -> 5644137ad HDFS-9941. Do not log StandbyException on NN, other minor logging fixes. Contributed by Arpit Agarwal. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5644137a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5644137a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5644137a Branch: refs/heads/trunk Commit: 5644137adad30c84e40d2c4719627b3aabc73628 Parents: f291d82 Author: Chris Nauroth Authored: Mon Mar 14 09:54:54 2016 -0700 Committer: Chris Nauroth Committed: Mon Mar 14 09:54:54 2016 -0700 ---------------------------------------------------------------------- .../BlockUnderConstructionFeature.java | 45 ++++++++++++-------- .../blockmanagement/DecommissionManager.java | 8 +++- .../hdfs/server/namenode/FSDirWriteFileOp.java | 18 +++++++- .../hdfs/server/namenode/NameNodeRpcServer.java | 4 ++ 4 files changed, 55 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5644137a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockUnderConstructionFeature.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockUnderConstructionFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockUnderConstructionFeature.java index b46b470..1a93033 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockUnderConstructionFeature.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockUnderConstructionFeature.java @@ -33,11 +33,13 @@ import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCSt */ public class BlockUnderConstructionFeature { private BlockUCState blockUCState; + private static final ReplicaUnderConstruction[] NO_REPLICAS = + new ReplicaUnderConstruction[0]; /** * Block replicas as assigned when the block was allocated. */ - private ReplicaUnderConstruction[] replicas; + private ReplicaUnderConstruction[] replicas = NO_REPLICAS; /** * Index of the primary data node doing the recovery. Useful for log @@ -120,7 +122,7 @@ public class BlockUnderConstructionFeature { } public int getNumExpectedLocations() { - return replicas == null ? 0 : replicas.length; + return replicas.length; } /** @@ -130,7 +132,7 @@ public class BlockUnderConstructionFeature { */ void updateStorageScheduledSize(BlockInfoStriped storedBlock) { assert storedBlock.getUnderConstructionFeature() == this; - if (replicas == null) { + if (replicas.length == 0) { return; } final int dataBlockNum = storedBlock.getDataBlockNum(); @@ -182,12 +184,10 @@ public class BlockUnderConstructionFeature { List getStaleReplicas(long genStamp) { List staleReplicas = new ArrayList<>(); - if (replicas != null) { - // Remove replicas with wrong gen stamp. The replica list is unchanged. - for (ReplicaUnderConstruction r : replicas) { - if (genStamp != r.getGenerationStamp()) { - staleReplicas.add(r); - } + // Remove replicas with wrong gen stamp. The replica list is unchanged. + for (ReplicaUnderConstruction r : replicas) { + if (genStamp != r.getGenerationStamp()) { + staleReplicas.add(r); } } return staleReplicas; @@ -201,7 +201,7 @@ public class BlockUnderConstructionFeature { public void initializeBlockRecovery(BlockInfo blockInfo, long recoveryId) { setBlockUCState(BlockUCState.UNDER_RECOVERY); blockRecoveryId = recoveryId; - if (replicas == null || replicas.length == 0) { + if (replicas.length == 0) { NameNode.blockStateChangeLog.warn("BLOCK*" + " BlockUnderConstructionFeature.initializeBlockRecovery:" + " No blocks found, lease removed."); @@ -252,7 +252,7 @@ public class BlockUnderConstructionFeature { /** Add the reported replica if it is not already in the replica list. */ void addReplicaIfNotPresent(DatanodeStorageInfo storage, Block reportedBlock, ReplicaState rState) { - if (replicas == null) { + if (replicas.length == 0) { replicas = new ReplicaUnderConstruction[1]; replicas[0] = new ReplicaUnderConstruction(reportedBlock, storage, rState); @@ -295,15 +295,24 @@ public class BlockUnderConstructionFeature { .append(", truncateBlock=").append(truncateBlock) .append(", primaryNodeIndex=").append(primaryNodeIndex) .append(", replicas=["); - if (replicas != null) { - int i = 0; - for (ReplicaUnderConstruction r : replicas) { - r.appendStringTo(sb); - if (++i < replicas.length) { - sb.append(", "); - } + int i = 0; + for (ReplicaUnderConstruction r : replicas) { + r.appendStringTo(sb); + if (++i < replicas.length) { + sb.append(", "); } } sb.append("]}"); } + + public void appendUCPartsConcise(StringBuilder sb) { + sb.append("replicas="); + int i = 0; + for (ReplicaUnderConstruction r : replicas) { + sb.append(r.getExpectedStorageLocation().getDatanodeDescriptor()); + if (++i < replicas.length) { + sb.append(", "); + } + } + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5644137a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java index 2a5d63c..480670a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java @@ -37,6 +37,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.server.namenode.INodeId; +import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.Namesystem; import org.apache.hadoop.hdfs.util.CyclicIteration; import org.apache.hadoop.util.ChunkedArrayList; @@ -280,6 +281,10 @@ public class DecommissionManager { BlockCollection bc, DatanodeDescriptor srcNode, NumberReplicas num, Iterable storages) { + if (!NameNode.blockStateChangeLog.isInfoEnabled()) { + return; + } + int curReplicas = num.liveReplicas(); int curExpectedReplicas = blockManager.getExpectedReplicaNum(block); StringBuilder nodeList = new StringBuilder(); @@ -288,7 +293,8 @@ public class DecommissionManager { nodeList.append(node); nodeList.append(" "); } - LOG.info("Block: " + block + ", Expected Replicas: " + NameNode.blockStateChangeLog.info( + "Block: " + block + ", Expected Replicas: " + curExpectedReplicas + ", live replicas: " + curReplicas + ", corrupt replicas: " + num.corruptReplicas() + ", decommissioned replicas: " + num.decommissioned() http://git-wip-us.apache.org/repos/asf/hadoop/blob/5644137a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java index cc08528..41fd869 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java @@ -848,10 +848,26 @@ class FSDirWriteFileOp { assert fsn.hasWriteLock(); BlockInfo b = addBlock(fsn.dir, src, inodesInPath, newBlock, targets, isStriped); - NameNode.stateChangeLog.info("BLOCK* allocate " + b + " for " + src); + logAllocatedBlock(src, b); DatanodeStorageInfo.incrementBlocksScheduled(targets); } + private static void logAllocatedBlock(String src, BlockInfo b) { + if (!NameNode.stateChangeLog.isInfoEnabled()) { + return; + } + StringBuilder sb = new StringBuilder(150); + sb.append("BLOCK* allocate "); + b.appendStringTo(sb); + sb.append(", "); + BlockUnderConstructionFeature uc = b.getUnderConstructionFeature(); + if (uc != null) { + uc.appendUCPartsConcise(sb); + } + sb.append(" for " + src); + NameNode.stateChangeLog.info(sb.toString()); + } + private static void setNewINodeStoragePolicy(BlockManager bm, INodeFile inode, INodesInPath iip, boolean isLazyPersist) throws IOException { http://git-wip-us.apache.org/repos/asf/hadoop/blob/5644137a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java index 6dff1bc..eb47580 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java @@ -163,6 +163,7 @@ import org.apache.hadoop.ipc.RetryCache; import org.apache.hadoop.ipc.RetryCache.CacheEntry; import org.apache.hadoop.ipc.RetryCache.CacheEntryWithPayload; import org.apache.hadoop.ipc.Server; +import org.apache.hadoop.ipc.StandbyException; import org.apache.hadoop.ipc.WritableRpcEngine; import org.apache.hadoop.ipc.RefreshRegistry; import org.apache.hadoop.ipc.RefreshResponse; @@ -494,6 +495,9 @@ class NameNodeRpcServer implements NamenodeProtocols { FSLimitException.PathComponentTooLongException.class, FSLimitException.MaxDirectoryItemsExceededException.class, UnresolvedPathException.class); + + clientRpcServer.addSuppressedLoggingExceptions(StandbyException.class); + clientRpcServer.setTracer(nn.tracer); if (serviceRpcServer != null) { serviceRpcServer.setTracer(nn.tracer);