hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aengin...@apache.org
Subject [40/50] [abbrv] hadoop git commit: YARN-5453. FairScheduler#update may skip update demand resource of child queue/app if current demand reached maxResource. (sandflee via kasha)
Date Fri, 11 Nov 2016 18:58:03 GMT
YARN-5453. FairScheduler#update may skip update demand resource of child queue/app if current
demand reached maxResource. (sandflee via kasha)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/86ac1ad9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/86ac1ad9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/86ac1ad9

Branch: refs/heads/HDFS-7240
Commit: 86ac1ad9fd65c7dd12278372b369de38dc4616db
Parents: c8bc7a8
Author: Karthik Kambatla <kasha@apache.org>
Authored: Wed Nov 9 23:44:02 2016 -0800
Committer: Karthik Kambatla <kasha@apache.org>
Committed: Wed Nov 9 23:44:02 2016 -0800

----------------------------------------------------------------------
 .../scheduler/fair/FSLeafQueue.java             | 15 +++----
 .../scheduler/fair/FSParentQueue.java           |  6 +--
 .../scheduler/fair/TestFairScheduler.java       | 41 ++++++++++++++++++++
 3 files changed, 48 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/86ac1ad9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index 9d5bbe5..c393759 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -268,20 +268,16 @@ public class FSLeafQueue extends FSQueue {
     readLock.lock();
     try {
       for (FSAppAttempt sched : runnableApps) {
-        if (Resources.equals(demand, maxShare)) {
-          break;
-        }
-        updateDemandForApp(sched, maxShare);
+        updateDemandForApp(sched);
       }
       for (FSAppAttempt sched : nonRunnableApps) {
-        if (Resources.equals(demand, maxShare)) {
-          break;
-        }
-        updateDemandForApp(sched, maxShare);
+        updateDemandForApp(sched);
       }
     } finally {
       readLock.unlock();
     }
+    // Cap demand to maxShare to limit allocation to maxShare
+    demand = Resources.componentwiseMin(demand, maxShare);
     if (LOG.isDebugEnabled()) {
       LOG.debug("The updated demand for " + getName() + " is " + demand
           + "; the max is " + maxShare);
@@ -290,7 +286,7 @@ public class FSLeafQueue extends FSQueue {
     }
   }
   
-  private void updateDemandForApp(FSAppAttempt sched, Resource maxRes) {
+  private void updateDemandForApp(FSAppAttempt sched) {
     sched.updateDemand();
     Resource toAdd = sched.getDemand();
     if (LOG.isDebugEnabled()) {
@@ -299,7 +295,6 @@ public class FSLeafQueue extends FSQueue {
           + demand);
     }
     demand = Resources.add(demand, toAdd);
-    demand = Resources.componentwiseMin(demand, maxRes);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hadoop/blob/86ac1ad9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
index d05390b..53ac8c9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
@@ -159,16 +159,14 @@ public class FSParentQueue extends FSQueue {
         childQueue.updateDemand();
         Resource toAdd = childQueue.getDemand();
         demand = Resources.add(demand, toAdd);
-        demand = Resources.componentwiseMin(demand, maxShare);
         if (LOG.isDebugEnabled()) {
           LOG.debug("Counting resource from " + childQueue.getName() + " " +
               toAdd + "; Total resource demand for " + getName() +
               " now " + demand);
         }
-        if (Resources.equals(demand, maxShare)) {
-          break;
-        }
       }
+      // Cap demand to maxShare to limit allocation to maxShare
+      demand = Resources.componentwiseMin(demand, maxShare);
     } finally {
       writeLock.unlock();
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/86ac1ad9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index f17726c..21d22c3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -4813,4 +4813,45 @@ public class TestFairScheduler extends FairSchedulerTestBase {
     assertEquals(0, metrics.getReservedMB());
     assertEquals(0, metrics.getReservedVirtualCores());
   }
+
+
+  @Test
+  public void testUpdateDemand() throws IOException {
+    scheduler.init(conf);
+    scheduler.start();
+    scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+    Resource maxResource = Resources.createResource(1024 * 8);
+
+    FSAppAttempt app1 = mock(FSAppAttempt.class);
+    Mockito.when(app1.getDemand()).thenReturn(maxResource);
+    FSAppAttempt app2 = mock(FSAppAttempt.class);
+    Mockito.when(app2.getDemand()).thenReturn(maxResource);
+
+    QueueManager queueManager = scheduler.getQueueManager();
+    FSParentQueue queue1 = queueManager.getParentQueue("queue1", true);
+
+    FSLeafQueue aQueue =
+        new FSLeafQueue("root.queue1.a", scheduler, queue1);
+    aQueue.setMaxShare(maxResource);
+    aQueue.addAppSchedulable(app1);
+
+    FSLeafQueue bQueue =
+        new FSLeafQueue("root.queue1.b", scheduler, queue1);
+    bQueue.setMaxShare(maxResource);
+    bQueue.addAppSchedulable(app2);
+
+    queue1.setMaxShare(maxResource);
+    queue1.addChildQueue(aQueue);
+    queue1.addChildQueue(bQueue);
+
+    queue1.updateDemand();
+
+    assertTrue("Demand is greater than max allowed ",
+        Resources.equals(queue1.getDemand(), maxResource));
+    assertTrue("Demand of child queue not updated ",
+        Resources.equals(aQueue.getDemand(), maxResource) &&
+        Resources.equals(bQueue.getDemand(), maxResource));
+  }
+
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message