Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 49CFF18BAA for ; Tue, 29 Sep 2015 20:30:32 +0000 (UTC) Received: (qmail 8226 invoked by uid 500); 29 Sep 2015 20:30:07 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 7968 invoked by uid 500); 29 Sep 2015 20:30:06 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 4505 invoked by uid 99); 29 Sep 2015 20:30:04 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 29 Sep 2015 20:30:04 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id ADCC0E0281; Tue, 29 Sep 2015 20:30:04 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: aengineer@apache.org To: common-commits@hadoop.apache.org Date: Tue, 29 Sep 2015 20:30:47 -0000 Message-Id: <13eedbc61ce448efa6628d260925c427@git.apache.org> In-Reply-To: <34d5a529b8034ef8aa87a88da0d49f7f@git.apache.org> References: <34d5a529b8034ef8aa87a88da0d49f7f@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [45/50] [abbrv] hadoop git commit: YARN-4066. Large number of queues choke fair scheduler. (Johan Gustavsson via kasha) YARN-4066. Large number of queues choke fair scheduler. (Johan Gustavsson via kasha) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a0b5a0a4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a0b5a0a4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a0b5a0a4 Branch: refs/heads/HDFS-7240 Commit: a0b5a0a419dfc07b7ac45c06b11b4c8dc7e79958 Parents: 715dbdd Author: Karthik Kambatla Authored: Tue Sep 29 07:55:34 2015 -0700 Committer: Karthik Kambatla Committed: Tue Sep 29 07:55:34 2015 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 + .../scheduler/fair/QueueManager.java | 80 ++++++++++++++------ 2 files changed, 60 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0b5a0a4/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index e9d04d3..43c501f 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -509,6 +509,9 @@ Release 2.8.0 - UNRELEASED YARN-3635. Refactored current queue mapping implementation in CapacityScheduler to use a generic PlacementManager framework. (Wangda Tan via jianhe) + YARN-4066. Large number of queues choke fair scheduler. + (Johan Gustavsson via kasha) + BUG FIXES YARN-3197. Confusing log generated by CapacityScheduler. (Varun Saxena http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0b5a0a4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java index 0092845..51a298b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java @@ -87,7 +87,19 @@ public class QueueManager { * could be referred to as just "parent1.queue2". */ public FSLeafQueue getLeafQueue(String name, boolean create) { - FSQueue queue = getQueue(name, create, FSQueueType.LEAF); + return getLeafQueue(name, create, true); + } + + public FSLeafQueue getLeafQueue( + String name, + boolean create, + boolean recomputeSteadyShares) { + FSQueue queue = getQueue( + name, + create, + FSQueueType.LEAF, + recomputeSteadyShares + ); if (queue instanceof FSParentQueue) { return null; } @@ -117,28 +129,46 @@ public class QueueManager { * could be referred to as just "parent1.queue2". */ public FSParentQueue getParentQueue(String name, boolean create) { - FSQueue queue = getQueue(name, create, FSQueueType.PARENT); + return getParentQueue(name, create, true); + } + + public FSParentQueue getParentQueue( + String name, + boolean create, + boolean recomputeSteadyShares) { + FSQueue queue = getQueue( + name, + create, + FSQueueType.PARENT, + recomputeSteadyShares + ); if (queue instanceof FSLeafQueue) { return null; } return (FSParentQueue) queue; } - - private FSQueue getQueue(String name, boolean create, FSQueueType queueType) { + + private FSQueue getQueue( + String name, + boolean create, + FSQueueType queueType, + boolean recomputeSteadyShares) { + boolean recompute = recomputeSteadyShares; name = ensureRootPrefix(name); + FSQueue queue; synchronized (queues) { - FSQueue queue = queues.get(name); + queue = queues.get(name); if (queue == null && create) { // if the queue doesn't exist,create it and return queue = createQueue(name, queueType); - - // Update steady fair share for all queues - if (queue != null) { - rootQueue.recomputeSteadyShares(); - } + } else { + recompute = false; } - return queue; } + if (recompute) { + rootQueue.recomputeSteadyShares(); + } + return queue; } /** @@ -376,21 +406,25 @@ public class QueueManager { public void updateAllocationConfiguration(AllocationConfiguration queueConf) { // Create leaf queues and the parent queues in a leaf's ancestry if they do not exist - for (String name : queueConf.getConfiguredQueues().get(FSQueueType.LEAF)) { - if (removeEmptyIncompatibleQueues(name, FSQueueType.LEAF)) { - getLeafQueue(name, true); + synchronized (queues) { + for (String name : queueConf.getConfiguredQueues().get( + FSQueueType.LEAF)) { + if (removeEmptyIncompatibleQueues(name, FSQueueType.LEAF)) { + getLeafQueue(name, true, false); + } } - } - - // At this point all leaves and 'parents with at least one child' would have been created. - // Now create parents with no configured leaf. - for (String name : queueConf.getConfiguredQueues().get( - FSQueueType.PARENT)) { - if (removeEmptyIncompatibleQueues(name, FSQueueType.PARENT)) { - getParentQueue(name, true); + // At this point all leaves and 'parents with + // at least one child' would have been created. + // Now create parents with no configured leaf. + for (String name : queueConf.getConfiguredQueues().get( + FSQueueType.PARENT)) { + if (removeEmptyIncompatibleQueues(name, FSQueueType.PARENT)) { + getParentQueue(name, true, false); + } } } - + rootQueue.recomputeSteadyShares(); + for (FSQueue queue : queues.values()) { // Update queue metrics FSQueueMetrics queueMetrics = queue.getMetrics();