hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cur...@apache.org
Subject hadoop git commit: YARN-3969. Allow jobs to be submitted to reservation that is active but does not have any allocations. (subru via curino)
Date Fri, 24 Jul 2015 15:04:03 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 b2a532f61 -> 6a71a0a23


YARN-3969. Allow jobs to be submitted to reservation that is active but does not have any
allocations. (subru via curino)

(cherry picked from commit 0fcb4a8cf2add3f112907ff4e833e2f04947b53e)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6a71a0a2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6a71a0a2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6a71a0a2

Branch: refs/heads/branch-2
Commit: 6a71a0a230b337c8d62cd116a9742c54f00728e9
Parents: b2a532f
Author: carlo curino <Carlo Curino>
Authored: Thu Jul 23 19:33:59 2015 -0700
Committer: carlo curino <Carlo Curino>
Committed: Thu Jul 23 19:36:54 2015 -0700

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                 |  3 +++
 .../scheduler/capacity/ReservationQueue.java    |  4 ---
 .../capacity/TestReservationQueue.java          | 26 +++++++++++---------
 3 files changed, 17 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a71a0a2/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index a150092..7b57203 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -816,6 +816,9 @@ Release 2.7.1 - 2015-07-06
     YARN-3850. NM fails to read files from full disks which can lead to
     container logs being lost and other issues (Varun Saxena via jlowe)
 
+    YARN-3969. Allow jobs to be submitted to reservation that is active 
+    but does not have any allocations. (subru via curino)
+
 Release 2.7.0 - 2015-04-20
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a71a0a2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java
index 4790cc7..976cf8c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java
@@ -39,12 +39,9 @@ public class ReservationQueue extends LeafQueue {
 
   private PlanQueue parent;
 
-  private int maxSystemApps;
-
   public ReservationQueue(CapacitySchedulerContext cs, String queueName,
       PlanQueue parent) throws IOException {
     super(cs, queueName, parent, null);
-    maxSystemApps = cs.getConfiguration().getMaximumSystemApplications();
     // the following parameters are common to all reservation in the plan
     updateQuotas(parent.getUserLimitForReservation(),
         parent.getUserLimitFactor(),
@@ -89,7 +86,6 @@ public class ReservationQueue extends LeafQueue {
     }
     setCapacity(capacity);
     setAbsoluteCapacity(getParent().getAbsoluteCapacity() * getCapacity());
-    setMaxApplications((int) (maxSystemApps * getAbsoluteCapacity()));
     // note: we currently set maxCapacity to capacity
     // this might be revised later
     setMaxCapacity(entitlement.getMaxCapacity());

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a71a0a2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java
index 4e6c73d..e23e93c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
 
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 import static org.mockito.Mockito.mock;
@@ -39,6 +40,7 @@ public class TestReservationQueue {
 
   CapacitySchedulerConfiguration csConf;
   CapacitySchedulerContext csContext;
+  final static int DEF_MAX_APPS = 10000;
   final static int GB = 1024;
   private final ResourceCalculator resourceCalculator =
       new DefaultResourceCalculator();
@@ -66,7 +68,13 @@ public class TestReservationQueue {
     // create a queue
     PlanQueue pq = new PlanQueue(csContext, "root", null, null);
     reservationQueue = new ReservationQueue(csContext, "a", pq);
+  }
 
+  private void validateReservationQueue(double capacity) {
+    assertTrue(" actual capacity: " + reservationQueue.getCapacity(),
+        reservationQueue.getCapacity() - capacity < CSQueueUtils.EPSILON);
+    assertEquals(reservationQueue.maxApplications, DEF_MAX_APPS);
+    assertEquals(reservationQueue.maxApplicationsPerUser, DEF_MAX_APPS);
   }
 
   @Test
@@ -74,25 +82,20 @@ public class TestReservationQueue {
 
     // verify that setting, adding, subtracting capacity works
     reservationQueue.setCapacity(1.0F);
-    assertTrue(" actual capacity: " + reservationQueue.getCapacity(),
-        reservationQueue.getCapacity() - 1 < CSQueueUtils.EPSILON);
+    validateReservationQueue(1);
     reservationQueue.setEntitlement(new QueueEntitlement(0.9f, 1f));
-    assertTrue(" actual capacity: " + reservationQueue.getCapacity(),
-        reservationQueue.getCapacity() - 0.9 < CSQueueUtils.EPSILON);
+    validateReservationQueue(0.9);
     reservationQueue.setEntitlement(new QueueEntitlement(1f, 1f));
-    assertTrue(" actual capacity: " + reservationQueue.getCapacity(),
-        reservationQueue.getCapacity() - 1 < CSQueueUtils.EPSILON);
+    validateReservationQueue(1);
     reservationQueue.setEntitlement(new QueueEntitlement(0f, 1f));
-    assertTrue(" actual capacity: " + reservationQueue.getCapacity(),
-        reservationQueue.getCapacity() < CSQueueUtils.EPSILON);
+    validateReservationQueue(0);
 
     try {
       reservationQueue.setEntitlement(new QueueEntitlement(1.1f, 1f));
       fail();
     } catch (SchedulerDynamicEditException iae) {
       // expected
-      assertTrue(" actual capacity: " + reservationQueue.getCapacity(),
-          reservationQueue.getCapacity() - 1 < CSQueueUtils.EPSILON);
+      validateReservationQueue(1);
     }
 
     try {
@@ -100,8 +103,7 @@ public class TestReservationQueue {
       fail();
     } catch (SchedulerDynamicEditException iae) {
       // expected
-      assertTrue(" actual capacity: " + reservationQueue.getCapacity(),
-          reservationQueue.getCapacity() - 1 < CSQueueUtils.EPSILON);
+      validateReservationQueue(1);
     }
 
   }


Mime
View raw message