hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From viraj...@apache.org
Subject [28/50] [abbrv] hadoop git commit: YARN-6439. Fix ReservationSystem creation of default ReservationQueue. (Carlo Curino via wangda)
Date Sat, 15 Apr 2017 19:10:49 GMT
YARN-6439. Fix ReservationSystem creation of default ReservationQueue. (Carlo Curino via wangda)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4d4ad0eb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4d4ad0eb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4d4ad0eb

Branch: refs/heads/HDFS-9806
Commit: 4d4ad0ebb76198c90c5317f19ce85a6696132c8f
Parents: 7d873c4
Author: Wangda Tan <wangda@apache.org>
Authored: Tue Apr 11 14:56:18 2017 -0700
Committer: Wangda Tan <wangda@apache.org>
Committed: Tue Apr 11 14:56:18 2017 -0700

----------------------------------------------------------------------
 .../capacity/CapacitySchedulerQueueManager.java | 20 ++++++++++++++++++++
 .../scheduler/capacity/PlanQueue.java           |  4 ++--
 .../TestCapacitySchedulerPlanFollower.java      |  7 +++++++
 .../TestFairSchedulerPlanFollower.java          |  4 ++++
 .../TestSchedulerPlanFollowerBase.java          |  9 ++++++++-
 .../TestCapacitySchedulerDynamicBehavior.java   | 11 +++++++++++
 6 files changed, 52 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d4ad0eb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
index c92c343..be6243d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
@@ -41,10 +41,13 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.security.Permission;
 import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationConstants;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueStateManager;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerQueueManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement;
 import org.apache.hadoop.yarn.server.resourcemanager.security.AppPriorityACLsManager;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -220,6 +223,23 @@ public class CapacitySchedulerQueueManager implements SchedulerQueueManager<
         queue =
             new PlanQueue(csContext, queueName, parent,
                 oldQueues.get(queueName));
+
+        //initializing the "internal" default queue, for SLS compatibility
+        String defReservationId =
+            queueName + ReservationConstants.DEFAULT_QUEUE_SUFFIX;
+
+        List<CSQueue> childQueues = new ArrayList<>();
+        ReservationQueue resQueue = new ReservationQueue(csContext,
+            defReservationId, (PlanQueue) queue);
+        try {
+          resQueue.setEntitlement(new QueueEntitlement(1.0f, 1.0f));
+        } catch (SchedulerDynamicEditException e) {
+          throw new IllegalStateException(e);
+        }
+        childQueues.add(resQueue);
+        ((PlanQueue) queue).setChildQueues(childQueues);
+        queues.put(defReservationId, resQueue);
+
       } else {
         queue =
             new LeafQueue(csContext, queueName, parent,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d4ad0eb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java
index a391f25..882262f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java
@@ -93,10 +93,10 @@ public class PlanQueue extends ParentQueue {
 
       PlanQueue newlyParsedParentQueue = (PlanQueue) newlyParsedQueue;
 
-      if (newlyParsedParentQueue.getChildQueues().size() > 0) {
+      if (newlyParsedParentQueue.getChildQueues().size() != 1) {
         throw new IOException(
             "Reservable Queue should not have sub-queues in the"
-                + "configuration");
+                + "configuration expect the default reservation queue");
       }
 
       // Set new configs

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d4ad0eb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java
index bccd5c3..4320d3d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java
@@ -154,6 +154,13 @@ public class TestCapacitySchedulerPlanFollower extends
   }
 
   @Override
+  protected void checkDefaultQueueBeforePlanFollowerRun(){
+    Queue defQ = getDefaultQueue();
+    Assert.assertEquals(0, getNumberOfApplications(defQ));
+    Assert.assertNotNull(defQ);
+  }
+
+  @Override
   protected Queue getDefaultQueue() {
     return cs.getQueue("dedicated" + ReservationConstants.DEFAULT_QUEUE_SUFFIX);
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d4ad0eb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestFairSchedulerPlanFollower.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestFairSchedulerPlanFollower.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestFairSchedulerPlanFollower.java
index 1b4e2f8..9561234 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestFairSchedulerPlanFollower.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestFairSchedulerPlanFollower.java
@@ -132,6 +132,10 @@ public class TestFairSchedulerPlanFollower extends
   }
 
   @Override
+  protected void checkDefaultQueueBeforePlanFollowerRun() {
+    Assert.assertNull(getDefaultQueue());
+  }
+  @Override
   protected void verifyCapacity(Queue defQ) {
     assertTrue(((FSQueue) defQ).getWeights().getWeight(ResourceType.MEMORY) > 0.9);
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d4ad0eb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSchedulerPlanFollowerBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSchedulerPlanFollowerBase.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSchedulerPlanFollowerBase.java
index b604799..c6cebd9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSchedulerPlanFollowerBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSchedulerPlanFollowerBase.java
@@ -89,6 +89,11 @@ public abstract class TestSchedulerPlanFollowerBase {
             "dedicated", 10, 10 + f2.length, ReservationSystemTestUtil
                 .generateAllocation(10L, 1L, f2), res, minAlloc), false));
 
+
+    // default reseration queue should exist before run of PlanFollower AND have
+    // no apps
+    checkDefaultQueueBeforePlanFollowerRun();
+
     AbstractSchedulerPlanFollower planFollower = createPlanFollower();
 
     when(mClock.getTime()).thenReturn(0L);
@@ -108,8 +113,8 @@ public abstract class TestSchedulerPlanFollowerBase {
         new AppAttemptAddedSchedulerEvent(appAttemptId_0, false);
     scheduler.handle(appAttemptAddedEvent);
 
-    // initial default reservation queue should have no apps
 
+    // initial default reservation queue should have no apps after first run
     Queue defQ = getDefaultQueue();
     Assert.assertEquals(0, getNumberOfApplications(defQ));
 
@@ -179,6 +184,8 @@ public abstract class TestSchedulerPlanFollowerBase {
     verifyCapacity(defQ);
   }
 
+  protected abstract void checkDefaultQueueBeforePlanFollowerRun();
+
   protected abstract Queue getReservationQueue(String reservationId);
 
   protected abstract void verifyCapacity(Queue defQ);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d4ad0eb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java
index ce3382f..483ba1b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java
@@ -76,6 +76,12 @@ public class TestCapacitySchedulerDynamicBehavior {
   public void testRefreshQueuesWithReservations() throws Exception {
     CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
 
+    //set default queue capacity to zero
+    ((ReservationQueue) cs
+            .getQueue("a" + ReservationConstants.DEFAULT_QUEUE_SUFFIX))
+            .setEntitlement(
+                    new QueueEntitlement(0f, 1f));
+
     // Test add one reservation dynamically and manually modify capacity
     ReservationQueue a1 =
         new ReservationQueue(cs, "a1", (PlanQueue) cs.getQueue("a"));
@@ -120,6 +126,11 @@ public class TestCapacitySchedulerDynamicBehavior {
     ReservationQueue a1 =
         new ReservationQueue(cs, "a1", (PlanQueue) cs.getQueue("a"));
     cs.addQueue(a1);
+    //set default queue capacity to zero
+    ((ReservationQueue) cs
+        .getQueue("a" + ReservationConstants.DEFAULT_QUEUE_SUFFIX))
+            .setEntitlement(
+                new QueueEntitlement(0f, 1f));
     a1.setEntitlement(new QueueEntitlement(A1_CAPACITY / 100, 1f));
 
     // Test add another reservation queue and use setEntitlement to modify


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message