Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 7D5C3200C78 for ; Thu, 13 Apr 2017 01:42:01 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 7C25A160BAC; Wed, 12 Apr 2017 23:42:01 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 7F248160BA9 for ; Thu, 13 Apr 2017 01:42:00 +0200 (CEST) Received: (qmail 51069 invoked by uid 500); 12 Apr 2017 23:41:49 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 49577 invoked by uid 99); 12 Apr 2017 23:41:49 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 12 Apr 2017 23:41:49 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 2948DE9809; Wed, 12 Apr 2017 23:41:49 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: aengineer@apache.org To: common-commits@hadoop.apache.org Date: Wed, 12 Apr 2017 23:42:21 -0000 Message-Id: In-Reply-To: <80589df660084b789597b791bc169975@git.apache.org> References: <80589df660084b789597b791bc169975@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [34/50] hadoop git commit: YARN-6439. Fix ReservationSystem creation of default ReservationQueue. (Carlo Curino via wangda) archived-at: Wed, 12 Apr 2017 23:42:01 -0000 YARN-6439. Fix ReservationSystem creation of default ReservationQueue. (Carlo Curino via wangda) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4d4ad0eb Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4d4ad0eb Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4d4ad0eb Branch: refs/heads/HDFS-7240 Commit: 4d4ad0ebb76198c90c5317f19ce85a6696132c8f Parents: 7d873c4 Author: Wangda Tan Authored: Tue Apr 11 14:56:18 2017 -0700 Committer: Wangda Tan Committed: Tue Apr 11 14:56:18 2017 -0700 ---------------------------------------------------------------------- .../capacity/CapacitySchedulerQueueManager.java | 20 ++++++++++++++++++++ .../scheduler/capacity/PlanQueue.java | 4 ++-- .../TestCapacitySchedulerPlanFollower.java | 7 +++++++ .../TestFairSchedulerPlanFollower.java | 4 ++++ .../TestSchedulerPlanFollowerBase.java | 9 ++++++++- .../TestCapacitySchedulerDynamicBehavior.java | 11 +++++++++++ 6 files changed, 52 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d4ad0eb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java index c92c343..be6243d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java @@ -41,10 +41,13 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.security.Permission; import org.apache.hadoop.yarn.security.YarnAuthorizationProvider; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationConstants; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueStateManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerQueueManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement; import org.apache.hadoop.yarn.server.resourcemanager.security.AppPriorityACLsManager; import com.google.common.annotations.VisibleForTesting; @@ -220,6 +223,23 @@ public class CapacitySchedulerQueueManager implements SchedulerQueueManager< queue = new PlanQueue(csContext, queueName, parent, oldQueues.get(queueName)); + + //initializing the "internal" default queue, for SLS compatibility + String defReservationId = + queueName + ReservationConstants.DEFAULT_QUEUE_SUFFIX; + + List childQueues = new ArrayList<>(); + ReservationQueue resQueue = new ReservationQueue(csContext, + defReservationId, (PlanQueue) queue); + try { + resQueue.setEntitlement(new QueueEntitlement(1.0f, 1.0f)); + } catch (SchedulerDynamicEditException e) { + throw new IllegalStateException(e); + } + childQueues.add(resQueue); + ((PlanQueue) queue).setChildQueues(childQueues); + queues.put(defReservationId, resQueue); + } else { queue = new LeafQueue(csContext, queueName, parent, http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d4ad0eb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java index a391f25..882262f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java @@ -93,10 +93,10 @@ public class PlanQueue extends ParentQueue { PlanQueue newlyParsedParentQueue = (PlanQueue) newlyParsedQueue; - if (newlyParsedParentQueue.getChildQueues().size() > 0) { + if (newlyParsedParentQueue.getChildQueues().size() != 1) { throw new IOException( "Reservable Queue should not have sub-queues in the" - + "configuration"); + + "configuration expect the default reservation queue"); } // Set new configs http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d4ad0eb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java index bccd5c3..4320d3d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java @@ -154,6 +154,13 @@ public class TestCapacitySchedulerPlanFollower extends } @Override + protected void checkDefaultQueueBeforePlanFollowerRun(){ + Queue defQ = getDefaultQueue(); + Assert.assertEquals(0, getNumberOfApplications(defQ)); + Assert.assertNotNull(defQ); + } + + @Override protected Queue getDefaultQueue() { return cs.getQueue("dedicated" + ReservationConstants.DEFAULT_QUEUE_SUFFIX); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d4ad0eb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestFairSchedulerPlanFollower.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestFairSchedulerPlanFollower.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestFairSchedulerPlanFollower.java index 1b4e2f8..9561234 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestFairSchedulerPlanFollower.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestFairSchedulerPlanFollower.java @@ -132,6 +132,10 @@ public class TestFairSchedulerPlanFollower extends } @Override + protected void checkDefaultQueueBeforePlanFollowerRun() { + Assert.assertNull(getDefaultQueue()); + } + @Override protected void verifyCapacity(Queue defQ) { assertTrue(((FSQueue) defQ).getWeights().getWeight(ResourceType.MEMORY) > 0.9); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d4ad0eb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSchedulerPlanFollowerBase.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSchedulerPlanFollowerBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSchedulerPlanFollowerBase.java index b604799..c6cebd9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSchedulerPlanFollowerBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSchedulerPlanFollowerBase.java @@ -89,6 +89,11 @@ public abstract class TestSchedulerPlanFollowerBase { "dedicated", 10, 10 + f2.length, ReservationSystemTestUtil .generateAllocation(10L, 1L, f2), res, minAlloc), false)); + + // default reseration queue should exist before run of PlanFollower AND have + // no apps + checkDefaultQueueBeforePlanFollowerRun(); + AbstractSchedulerPlanFollower planFollower = createPlanFollower(); when(mClock.getTime()).thenReturn(0L); @@ -108,8 +113,8 @@ public abstract class TestSchedulerPlanFollowerBase { new AppAttemptAddedSchedulerEvent(appAttemptId_0, false); scheduler.handle(appAttemptAddedEvent); - // initial default reservation queue should have no apps + // initial default reservation queue should have no apps after first run Queue defQ = getDefaultQueue(); Assert.assertEquals(0, getNumberOfApplications(defQ)); @@ -179,6 +184,8 @@ public abstract class TestSchedulerPlanFollowerBase { verifyCapacity(defQ); } + protected abstract void checkDefaultQueueBeforePlanFollowerRun(); + protected abstract Queue getReservationQueue(String reservationId); protected abstract void verifyCapacity(Queue defQ); http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d4ad0eb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java index ce3382f..483ba1b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java @@ -76,6 +76,12 @@ public class TestCapacitySchedulerDynamicBehavior { public void testRefreshQueuesWithReservations() throws Exception { CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); + //set default queue capacity to zero + ((ReservationQueue) cs + .getQueue("a" + ReservationConstants.DEFAULT_QUEUE_SUFFIX)) + .setEntitlement( + new QueueEntitlement(0f, 1f)); + // Test add one reservation dynamically and manually modify capacity ReservationQueue a1 = new ReservationQueue(cs, "a1", (PlanQueue) cs.getQueue("a")); @@ -120,6 +126,11 @@ public class TestCapacitySchedulerDynamicBehavior { ReservationQueue a1 = new ReservationQueue(cs, "a1", (PlanQueue) cs.getQueue("a")); cs.addQueue(a1); + //set default queue capacity to zero + ((ReservationQueue) cs + .getQueue("a" + ReservationConstants.DEFAULT_QUEUE_SUFFIX)) + .setEntitlement( + new QueueEntitlement(0f, 1f)); a1.setEntitlement(new QueueEntitlement(A1_CAPACITY / 100, 1f)); // Test add another reservation queue and use setEntitlement to modify --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org