hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gif...@apache.org
Subject hadoop git commit: HDFS-14102. Performance improvement in BlockPlacementPolicyDefault. Contributed by Beluga Behr.
Date Wed, 28 Nov 2018 19:33:50 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk 4ca3a6b21 -> 300f560fc


HDFS-14102. Performance improvement in BlockPlacementPolicyDefault. Contributed by Beluga
Behr.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/300f560f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/300f560f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/300f560f

Branch: refs/heads/trunk
Commit: 300f560fcca879d212eadffaa59fbbb7017c9a3f
Parents: 4ca3a6b
Author: Giovanni Matteo Fumarola <gifuma@apache.org>
Authored: Wed Nov 28 11:33:22 2018 -0800
Committer: Giovanni Matteo Fumarola <gifuma@apache.org>
Committed: Wed Nov 28 11:33:22 2018 -0800

----------------------------------------------------------------------
 .../BlockPlacementPolicyDefault.java            | 22 ++++++++++++--------
 1 file changed, 13 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/300f560f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index d396845..5b8a8b4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -71,6 +71,9 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
       CHOOSE_RANDOM_REASONS = ThreadLocal
       .withInitial(() -> new HashMap<NodeNotChosenReason, Integer>());
 
+  private static final BlockPlacementStatus ONE_RACK_PLACEMENT =
+      new BlockPlacementStatusDefault(1, 1, 1);
+
   private enum NodeNotChosenReason {
     NOT_IN_SERVICE("the node is not in service"),
     NODE_STALE("the node is stale"),
@@ -1029,22 +1032,23 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy
{
   @Override
   public BlockPlacementStatus verifyBlockPlacement(DatanodeInfo[] locs,
       int numberOfReplicas) {
-    if (locs == null)
+    if (locs == null) {
       locs = DatanodeDescriptor.EMPTY_ARRAY;
+    }
     if (!clusterMap.hasClusterEverBeenMultiRack()) {
       // only one rack
-      return new BlockPlacementStatusDefault(1, 1, 1);
+      return ONE_RACK_PLACEMENT;
     }
-    int minRacks = 2;
-    minRacks = Math.min(minRacks, numberOfReplicas);
+    final int minRacks = Math.min(2, numberOfReplicas);
     // 1. Check that all locations are different.
     // 2. Count locations on different racks.
-    Set<String> racks = new TreeSet<>();
-    for (DatanodeInfo dn : locs)
-      racks.add(dn.getNetworkLocation());
-    return new BlockPlacementStatusDefault(racks.size(), minRacks,
-        clusterMap.getNumOfRacks());
+    final long rackCount = Arrays.asList(locs).stream()
+        .map(dn -> dn.getNetworkLocation()).distinct().count();
+
+    return new BlockPlacementStatusDefault(Math.toIntExact(rackCount),
+        minRacks, clusterMap.getNumOfRacks());
   }
+
   /**
    * Decide whether deleting the specified replica of the block still makes
    * the block conform to the configured block placement policy.


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message