Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id C80391868A for ; Wed, 30 Sep 2015 15:42:30 +0000 (UTC) Received: (qmail 18224 invoked by uid 500); 30 Sep 2015 15:42:04 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 17509 invoked by uid 500); 30 Sep 2015 15:42:03 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 15058 invoked by uid 99); 30 Sep 2015 15:42:02 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 30 Sep 2015 15:42:02 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 22EC4E0A51; Wed, 30 Sep 2015 15:42:02 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: wang@apache.org To: common-commits@hadoop.apache.org Date: Wed, 30 Sep 2015 15:42:36 -0000 Message-Id: <6e2612b3c0fc448e887999e7c078f7df@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [36/50] [abbrv] hadoop git commit: HDFS-8920. Erasure Coding: when recovering lost blocks, logs can be too verbose and hurt performance. Contributed by Rui Li HDFS-8920. Erasure Coding: when recovering lost blocks, logs can be too verbose and hurt performance. Contributed by Rui Li Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7bff8ca1 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7bff8ca1 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7bff8ca1 Branch: refs/heads/trunk Commit: 7bff8ca1c872ea534a96cbbc5f70134574e289ce Parents: 1080c37 Author: Kai Zheng Authored: Wed Sep 23 14:13:26 2015 +0800 Committer: Zhe Zhang Committed: Tue Sep 22 13:27:24 2015 -0700 ---------------------------------------------------------------------- .../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt | 3 ++ .../org/apache/hadoop/hdfs/DFSInputStream.java | 15 +++++++-- .../hadoop/hdfs/DFSStripedInputStream.java | 34 ++++++++++++++++++++ 3 files changed, 49 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bff8ca1/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt index 0e21d22..b79ce64 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt @@ -444,3 +444,6 @@ HDFS-9091. Erasure Coding: Provide DistributedFilesystem API to getAllErasureCodingPolicies. (Rakesh R via zhz) + + HDFS-8920. Erasure Coding: when recovering lost blocks, logs can be too + verbose and hurt performance. (Rui Li via Kai Zheng) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bff8ca1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java index a5911cc..385ba4b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java @@ -1057,9 +1057,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead, } } if (chosenNode == null) { - DFSClient.LOG.warn("No live nodes contain block " + block.getBlock() + - " after checking nodes = " + Arrays.toString(nodes) + - ", ignoredNodes = " + ignoredNodes); + reportLostBlock(block, ignoredNodes); return null; } final String dnAddr = @@ -1071,6 +1069,17 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead, return new DNAddrPair(chosenNode, targetAddr, storageType); } + /** + * Warn the user of a lost block + */ + protected void reportLostBlock(LocatedBlock lostBlock, + Collection ignoredNodes) { + DatanodeInfo[] nodes = lostBlock.getLocations(); + DFSClient.LOG.warn("No live nodes contain block " + lostBlock.getBlock() + + " after checking nodes = " + Arrays.toString(nodes) + + ", ignoredNodes = " + ignoredNodes); + } + private static String getBestNodeDNAddrPairErrorString( DatanodeInfo nodes[], AbstractMap deadNodes, Collection ignoredNodes) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bff8ca1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java index b7c22c4..131a552 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java @@ -45,8 +45,11 @@ import java.io.EOFException; import java.io.IOException; import java.io.InterruptedIOException; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.EnumSet; +import java.util.List; import java.util.Set; import java.util.Collection; import java.util.Map; @@ -154,6 +157,17 @@ public class DFSStripedInputStream extends DFSInputStream { private StripeRange curStripeRange; private final CompletionService readingService; + /** + * When warning the user of a lost block in striping mode, we remember the + * dead nodes we've logged. All other striping blocks on these nodes can be + * considered lost too, and we don't want to log a warning for each of them. + * This is to prevent the log from being too verbose. Refer to HDFS-8920. + * + * To minimize the overhead, we only store the datanodeUuid in this set + */ + private final Set warnedNodes = Collections.newSetFromMap( + new ConcurrentHashMap()); + DFSStripedInputStream(DFSClient dfsClient, String src, boolean verifyChecksum, ErasureCodingPolicy ecPolicy, LocatedBlocks locatedBlocks) throws IOException { @@ -527,6 +541,26 @@ public class DFSStripedInputStream extends DFSInputStream { } } + @Override + protected void reportLostBlock(LocatedBlock lostBlock, + Collection ignoredNodes) { + DatanodeInfo[] nodes = lostBlock.getLocations(); + if (nodes != null && nodes.length > 0) { + List dnUUIDs = new ArrayList<>(); + for (DatanodeInfo node : nodes) { + dnUUIDs.add(node.getDatanodeUuid()); + } + if (!warnedNodes.containsAll(dnUUIDs)) { + DFSClient.LOG.warn(Arrays.toString(nodes) + " are unavailable and " + + "all striping blocks on them are lost. " + + "IgnoredNodes = " + ignoredNodes); + warnedNodes.addAll(dnUUIDs); + } + } else { + super.reportLostBlock(lostBlock, ignoredNodes); + } + } + /** * The reader for reading a complete {@link AlignedStripe}. Note that an * {@link AlignedStripe} may cross multiple stripes with cellSize width.