hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From szets...@apache.org
Subject hadoop git commit: HDFS-8278. When computing max-size-to-move in Balancer, count only the storage with remaining >= default block size.
Date Tue, 18 Aug 2015 01:43:11 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 076c68878 -> 9ab89a926


HDFS-8278. When computing max-size-to-move in Balancer, count only the storage with remaining
>= default block size.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9ab89a92
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9ab89a92
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9ab89a92

Branch: refs/heads/branch-2
Commit: 9ab89a9266acaa33007c888213d7b2cc2110eb60
Parents: 076c688
Author: Tsz-Wo Nicholas Sze <szetszwo@hortonworks.com>
Authored: Mon Aug 17 17:55:25 2015 -0700
Committer: Tsz-Wo Nicholas Sze <szetszwo@hortonworks.com>
Committed: Mon Aug 17 18:42:41 2015 -0700

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt         |  3 +++
 .../hadoop/hdfs/server/balancer/Balancer.java       | 16 +++++++++++-----
 2 files changed, 14 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ab89a92/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 31b197a..24ce942 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -458,6 +458,9 @@ Release 2.8.0 - UNRELEASED
 
     HDFS-8880. NameNode metrics logging. (Arpit Agarwal)
 
+    HDFS-8278. When computing max-size-to-move in Balancer, count only the
+    storage with remaining >= default block size.  (szetszwo)
+
   OPTIMIZATIONS
 
     HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ab89a92/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
index 4f3f18e..16945df 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
@@ -192,6 +192,7 @@ public class Balancer {
   private final boolean runDuringUpgrade;
   private final double threshold;
   private final long maxSizeToMove;
+  private final long defaultBlockSize;
 
   // all data node lists
   private final Collection<Source> overUtilized = new LinkedList<Source>();
@@ -270,6 +271,9 @@ public class Balancer {
     this.maxSizeToMove = getLong(conf,
         DFSConfigKeys.DFS_BALANCER_MAX_SIZE_TO_MOVE_KEY,
         DFSConfigKeys.DFS_BALANCER_MAX_SIZE_TO_MOVE_DEFAULT);
+    this.defaultBlockSize = getLong(conf,
+        DFSConfigKeys.DFS_BLOCK_SIZE_KEY,
+        DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT);
   }
   
   private static long getCapacity(DatanodeStorageReport report, StorageType t) {
@@ -282,11 +286,13 @@ public class Balancer {
     return capacity;
   }
 
-  private static long getRemaining(DatanodeStorageReport report, StorageType t) {
+  private long getRemaining(DatanodeStorageReport report, StorageType t) {
     long remaining = 0L;
     for(StorageReport r : report.getStorageReports()) {
       if (r.getStorage().getStorageType() == t) {
-        remaining += r.getRemaining();
+        if (r.getRemaining() >= defaultBlockSize) {
+          remaining += r.getRemaining();
+        }
       }
     }
     return remaining;
@@ -323,7 +329,7 @@ public class Balancer {
         final double utilizationDiff = utilization - policy.getAvgUtilization(t);
         final double thresholdDiff = Math.abs(utilizationDiff) - threshold;
         final long maxSize2Move = computeMaxSize2Move(capacity,
-            getRemaining(r, t), utilizationDiff, threshold, maxSizeToMove);
+            getRemaining(r, t), utilizationDiff, maxSizeToMove);
 
         final StorageGroup g;
         if (utilizationDiff > 0) {
@@ -360,8 +366,8 @@ public class Balancer {
   }
 
   private static long computeMaxSize2Move(final long capacity, final long remaining,
-      final double utilizationDiff, final double threshold, final long max) {
-    final double diff = Math.min(threshold, Math.abs(utilizationDiff));
+      final double utilizationDiff, final long max) {
+    final double diff = Math.abs(utilizationDiff);
     long maxSizeToMove = percentage2bytes(diff, capacity);
     if (utilizationDiff < 0) {
       maxSizeToMove = Math.min(remaining, maxSizeToMove);


Mime
View raw message