Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 6B657188EB for ; Fri, 29 May 2015 22:17:28 +0000 (UTC) Received: (qmail 53075 invoked by uid 500); 29 May 2015 22:17:28 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 53007 invoked by uid 500); 29 May 2015 22:17:28 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 52998 invoked by uid 99); 29 May 2015 22:17:28 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 29 May 2015 22:17:28 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 15928E0329; Fri, 29 May 2015 22:17:28 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: kasha@apache.org To: common-commits@hadoop.apache.org Message-Id: <56ecdc6331134ce3a4b7f086ce68c034@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: YARN-3547. FairScheduler: Apps that have no resource demand should not participate scheduling. (Xianyin Xin via kasha) Date: Fri, 29 May 2015 22:17:28 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/trunk 7673d4f20 -> 3ae2a6250 YARN-3547. FairScheduler: Apps that have no resource demand should not participate scheduling. (Xianyin Xin via kasha) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3ae2a625 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3ae2a625 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3ae2a625 Branch: refs/heads/trunk Commit: 3ae2a625018bc8cf431aa19da5bf8fe4ef8c1ad4 Parents: 7673d4f Author: Karthik Kambatla Authored: Fri May 29 15:17:02 2015 -0700 Committer: Karthik Kambatla Committed: Fri May 29 15:17:22 2015 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 ++ .../scheduler/fair/FSLeafQueue.java | 43 ++++++++++---------- 2 files changed, 24 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ae2a625/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index c1f1044..962d7ba 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -291,6 +291,9 @@ Release 2.8.0 - UNRELEASED YARN-3006. Improve the error message when attempting manual failover with auto-failover enabled. (Akira AJISAKA via wangda) + YARN-3547. FairScheduler: Apps that have no resource demand should not participate + scheduling. (Xianyin Xin via kasha) + BUG FIXES YARN-3197. Confusing log generated by CapacityScheduler. (Varun Saxena http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ae2a625/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java index 04dbd2f..6779a1b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.TreeSet; import com.google.common.annotations.VisibleForTesting; import org.apache.commons.logging.Log; @@ -314,35 +315,33 @@ public class FSLeafQueue extends FSQueue { return assigned; } - Comparator comparator = policy.getComparator(); - writeLock.lock(); - try { - Collections.sort(runnableApps, comparator); - } finally { - writeLock.unlock(); - } - // Release write lock here for better performance and avoiding deadlocks. - // runnableApps can be in unsorted state because of this section, - // but we can accept it in practice since the probability is low. + // Apps that have resource demands. + TreeSet pendingForResourceApps = + new TreeSet(policy.getComparator()); readLock.lock(); try { - for (FSAppAttempt sched : runnableApps) { - if (SchedulerAppUtils.isBlacklisted(sched, node, LOG)) { - continue; - } - - assigned = sched.assignContainer(node); - if (!assigned.equals(Resources.none())) { - if (LOG.isDebugEnabled()) { - LOG.debug("Assigned container in queue:" + getName() + " " + - "container:" + assigned); - } - break; + for (FSAppAttempt app : runnableApps) { + Resource pending = app.getAppAttemptResourceUsage().getPending(); + if (!pending.equals(Resources.none())) { + pendingForResourceApps.add(app); } } } finally { readLock.unlock(); } + for (FSAppAttempt sched : pendingForResourceApps) { + if (SchedulerAppUtils.isBlacklisted(sched, node, LOG)) { + continue; + } + assigned = sched.assignContainer(node); + if (!assigned.equals(Resources.none())) { + if (LOG.isDebugEnabled()) { + LOG.debug("Assigned container in queue:" + getName() + " " + + "container:" + assigned); + } + break; + } + } return assigned; }