hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From su...@apache.org
Subject [06/50] [abbrv] hadoop git commit: YARN-6264. AM not launched when a single vcore is available on the cluster. (Yufei Gu via kasha)
Date Sat, 11 Mar 2017 02:22:26 GMT
YARN-6264. AM not launched when a single vcore is available on the cluster. (Yufei Gu via kasha)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a96afae1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a96afae1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a96afae1

Branch: refs/heads/YARN-2915
Commit: a96afae125ba02fb4480542d3fb0891623ee4c37
Parents: c5ee7fde
Author: Karthik Kambatla <kasha@cloudera.com>
Authored: Thu Mar 9 23:11:54 2017 -0800
Committer: Karthik Kambatla <kasha@cloudera.com>
Committed: Thu Mar 9 23:11:54 2017 -0800

----------------------------------------------------------------------
 .../hadoop/yarn/util/resource/Resources.java    |  7 +++++
 .../yarn/util/resource/TestResources.java       | 24 ++++++++++++++++-
 .../scheduler/fair/FSLeafQueue.java             |  3 ++-
 .../scheduler/fair/TestFairScheduler.java       | 28 ++++++++++----------
 4 files changed, 46 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a96afae1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
index 57b3a46..7020300 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
@@ -242,6 +242,13 @@ public class Resources {
     out.setVirtualCores((int)(lhs.getVirtualCores() * by));
     return out;
   }
+
+  public static Resource multiplyAndRoundUp(Resource lhs, double by) {
+    Resource out = clone(lhs);
+    out.setMemorySize((long)Math.ceil(lhs.getMemorySize() * by));
+    out.setVirtualCores((int)Math.ceil(lhs.getVirtualCores() * by));
+    return out;
+  }
   
   public static Resource normalize(
       ResourceCalculator calculator, Resource lhs, Resource min,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a96afae1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java
index 057214b..f8570a8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.yarn.util.resource;
 
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
 public class TestResources {
@@ -46,5 +48,25 @@ public class TestResources {
     assertTrue(Resources.none().compareTo(
         createResource(0, 1)) < 0);
   }
-  
+
+  @Test
+  public void testMultipleRoundUp() {
+    final double by = 0.5;
+    final String memoryErrorMsg = "Invalid memory size.";
+    final String vcoreErrorMsg = "Invalid virtual core number.";
+    Resource resource = Resources.createResource(1, 1);
+    Resource result = Resources.multiplyAndRoundUp(resource, by);
+    assertEquals(memoryErrorMsg, result.getMemorySize(), 1);
+    assertEquals(vcoreErrorMsg, result.getVirtualCores(), 1);
+
+    resource = Resources.createResource(2, 2);
+    result = Resources.multiplyAndRoundUp(resource, by);
+    assertEquals(memoryErrorMsg, result.getMemorySize(), 1);
+    assertEquals(vcoreErrorMsg, result.getVirtualCores(), 1);
+
+    resource = Resources.createResource(0, 0);
+    result = Resources.multiplyAndRoundUp(resource, by);
+    assertEquals(memoryErrorMsg, result.getMemorySize(), 0);
+    assertEquals(vcoreErrorMsg, result.getVirtualCores(), 0);
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a96afae1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index aad2916..5be4b36 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -515,7 +515,8 @@ public class FSLeafQueue extends FSQueue {
           getMaxShare().getVirtualCores()));
     }
 
-    return Resources.multiply(maxResource, maxAMShare);
+    // Round up to allow AM to run when there is only one vcore on the cluster
+    return Resources.multiplyAndRoundUp(maxResource, maxAMShare);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a96afae1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 028eea6..cdf0d81 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -659,15 +659,13 @@ public class TestFairScheduler extends FairSchedulerTestBase {
     // case, we use maxShare, since it is smaller than available resource.
     assertEquals("QueueFSZeroWithMax's fair share should be zero",
         0, queueFSZeroWithMax.getFairShare().getMemorySize());
+    Resource expectedAMResource = Resources.multiplyAndRoundUp(
+        queueFSZeroWithMax.getMaxShare(), queueFSZeroWithMax.getMaxAMShare());
     assertEquals("QueueFSZeroWithMax's maximum AM resource should be "
-        + "maxShare * maxAMShare",
-        (long)(queueFSZeroWithMax.getMaxShare().getMemorySize() *
-            queueFSZeroWithMax.getMaxAMShare()),
+        + "maxShare * maxAMShare", expectedAMResource.getMemorySize(),
         queueFSZeroWithMax.getMetrics().getMaxAMShareMB());
     assertEquals("QueueFSZeroWithMax's maximum AM resource should be "
-        + "maxShare * maxAMShare",
-        (long)(queueFSZeroWithMax.getMaxShare().getVirtualCores() *
-            queueFSZeroWithMax.getMaxAMShare()),
+        + "maxShare * maxAMShare", expectedAMResource.getVirtualCores(),
         queueFSZeroWithMax.getMetrics().getMaxAMShareVCores());
     assertEquals("QueueFSZeroWithMax's AM resource usage should be the same to "
         + "AM resource request",
@@ -689,17 +687,19 @@ public class TestFairScheduler extends FairSchedulerTestBase {
     // the min(maxShare, available resource) to compute maxAMShare, in this
     // case, we use available resource since it is smaller than the
     // default maxShare.
+    expectedAMResource = Resources.multiplyAndRoundUp(
+        Resources.createResource(memCapacity - amResource.getMemorySize(),
+            cpuCapacity - amResource.getVirtualCores()),
+        queueFSZeroWithAVL.getMaxAMShare());
     assertEquals("QueueFSZeroWithAVL's fair share should be zero",
         0, queueFSZeroWithAVL.getFairShare().getMemorySize());
     assertEquals("QueueFSZeroWithAVL's maximum AM resource should be "
         + " available resource * maxAMShare",
-        (long) ((memCapacity - amResource.getMemorySize()) *
-        queueFSZeroWithAVL.getMaxAMShare()),
+        expectedAMResource.getMemorySize(),
         queueFSZeroWithAVL.getMetrics().getMaxAMShareMB());
     assertEquals("QueueFSZeroWithAVL's maximum AM resource should be "
         + " available resource * maxAMShare",
-        (long) ((cpuCapacity - amResource.getVirtualCores()) *
-        queueFSZeroWithAVL.getMaxAMShare()),
+        expectedAMResource.getVirtualCores(),
         queueFSZeroWithAVL.getMetrics().getMaxAMShareVCores());
     assertEquals("QueueFSZeroWithMax's AM resource usage should be the same to "
         + "AM resource request",
@@ -721,13 +721,13 @@ public class TestFairScheduler extends FairSchedulerTestBase {
     // fair share to compute maxAMShare
     assertNotEquals("QueueFSNonZero's fair share shouldn't be zero",
         0, queueFSNonZero.getFairShare().getMemorySize());
+    expectedAMResource = Resources.multiplyAndRoundUp(
+        queueFSNonZero.getFairShare(), queueFSNonZero.getMaxAMShare());
     assertEquals("QueueFSNonZero's maximum AM resource should be "
-        + " fair share * maxAMShare",
-        (long)(memCapacity * queueFSNonZero.getMaxAMShare()),
+        + " fair share * maxAMShare", expectedAMResource.getMemorySize(),
         queueFSNonZero.getMetrics().getMaxAMShareMB());
     assertEquals("QueueFSNonZero's maximum AM resource should be "
-        + " fair share * maxAMShare",
-        (long)(cpuCapacity * queueFSNonZero.getMaxAMShare()),
+        + " fair share * maxAMShare", expectedAMResource.getVirtualCores(),
         queueFSNonZero.getMetrics().getMaxAMShareVCores());
     assertEquals("QueueFSNonZero's AM resource usage should be the same to "
         + "AM resource request",


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message