Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id CFD37183BA for ; Mon, 1 Feb 2016 08:04:01 +0000 (UTC) Received: (qmail 34495 invoked by uid 500); 1 Feb 2016 08:03:45 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 34426 invoked by uid 500); 1 Feb 2016 08:03:45 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 34412 invoked by uid 99); 1 Feb 2016 08:03:45 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 01 Feb 2016 08:03:45 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 744C1E031B; Mon, 1 Feb 2016 08:03:45 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: vinayakumarb@apache.org To: common-commits@hadoop.apache.org Date: Mon, 01 Feb 2016 08:03:45 -0000 Message-Id: <328094c873c84e7c988edab538f28ebf@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [1/3] hadoop git commit: HDFS-9566. Remove expensive 'BlocksMap#getStorages(Block b, final DatanodeStorage.State state)' method (Contributed by Daryn Sharp) Repository: hadoop Updated Branches: refs/heads/branch-2 676f21559 -> 71374cca9 refs/heads/branch-2.8 a9916057e -> 6eae76f7a refs/heads/trunk 2673cbaf5 -> e418bd1fb HDFS-9566. Remove expensive 'BlocksMap#getStorages(Block b, final DatanodeStorage.State state)' method (Contributed by Daryn Sharp) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e418bd1f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e418bd1f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e418bd1f Branch: refs/heads/trunk Commit: e418bd1fb0568ce7ae22f588fea2dd9c95567383 Parents: 2673cba Author: Vinayakumar B Authored: Mon Feb 1 13:24:05 2016 +0530 Committer: Vinayakumar B Committed: Mon Feb 1 13:24:05 2016 +0530 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../server/blockmanagement/BlockManager.java | 16 ++++++++++++---- .../hdfs/server/blockmanagement/BlocksMap.java | 20 -------------------- 3 files changed, 15 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e418bd1f/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index f7487fe..432e686 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -2655,6 +2655,9 @@ Release 2.8.0 - UNRELEASED HDFS-9682. Fix a typo "aplication" in HttpFS document. (Weiwei Yang via aajisaka) + HDFS-9566. Remove expensive 'BlocksMap#getStorages(Block b, final + DatanodeStorage.State state)' method (Daryn Sharp via vinayakumarb) + Release 2.7.3 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/e418bd1f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index a76429e..587e6b6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -1212,8 +1212,10 @@ public class BlockManager implements BlockStatsMXBean { return; } StringBuilder datanodes = new StringBuilder(); - for(DatanodeStorageInfo storage : blocksMap.getStorages(storedBlock, - State.NORMAL)) { + for (DatanodeStorageInfo storage : blocksMap.getStorages(storedBlock)) { + if (storage.getState() != State.NORMAL) { + continue; + } final DatanodeDescriptor node = storage.getDatanodeDescriptor(); final Block b = getBlockOnStorage(storedBlock, storage); if (b != null) { @@ -3164,7 +3166,10 @@ public class BlockManager implements BlockStatsMXBean { Collection nonExcess = new ArrayList<>(); Collection corruptNodes = corruptReplicas .getNodes(block); - for(DatanodeStorageInfo storage : blocksMap.getStorages(block, State.NORMAL)) { + for (DatanodeStorageInfo storage : blocksMap.getStorages(block)) { + if (storage.getState() != State.NORMAL) { + continue; + } final DatanodeDescriptor cur = storage.getDatanodeDescriptor(); if (storage.areBlockContentsStale()) { LOG.trace("BLOCK* processOverReplicatedBlock: Postponing {}" @@ -3665,7 +3670,10 @@ public class BlockManager implements BlockStatsMXBean { // else proceed with fast case int live = 0; Collection nodesCorrupt = corruptReplicas.getNodes(b); - for(DatanodeStorageInfo storage : blocksMap.getStorages(b, State.NORMAL)) { + for (DatanodeStorageInfo storage : blocksMap.getStorages(b)) { + if (storage.getState() != State.NORMAL) { + continue; + } final DatanodeDescriptor node = storage.getDatanodeDescriptor(); if ((nodesCorrupt == null) || (!nodesCorrupt.contains(node))) live++; http://git-wip-us.apache.org/repos/asf/hadoop/blob/e418bd1f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java index ed05e3a..47a21fe 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java @@ -21,13 +21,9 @@ import java.util.Iterator; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.server.namenode.INodeId; -import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.apache.hadoop.util.GSet; import org.apache.hadoop.util.LightWeightGSet; -import com.google.common.base.Predicate; -import com.google.common.collect.Iterables; - /** * This class maintains the map from a block to its metadata. * block's metadata currently includes blockCollection it belongs to and @@ -150,22 +146,6 @@ class BlocksMap { } /** - * Searches for the block in the BlocksMap and - * returns {@link Iterable} of the storages the block belongs to - * that are of the given {@link DatanodeStorage.State state}. - * - * @param state DatanodeStorage state by which to filter the returned Iterable - */ - Iterable getStorages(Block b, final DatanodeStorage.State state) { - return Iterables.filter(getStorages(blocks.get(b)), new Predicate() { - @Override - public boolean apply(DatanodeStorageInfo storage) { - return storage.getState() == state; - } - }); - } - - /** * For a block that has already been retrieved from the BlocksMap * returns {@link Iterable} of the storages the block belongs to. */