Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 58D0A18D67 for ; Fri, 29 May 2015 00:05:45 +0000 (UTC) Received: (qmail 79620 invoked by uid 500); 29 May 2015 00:05:45 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 79558 invoked by uid 500); 29 May 2015 00:05:45 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 79549 invoked by uid 99); 29 May 2015 00:05:45 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 29 May 2015 00:05:45 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id DA99AE10D1; Fri, 29 May 2015 00:05:44 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: wangda@apache.org To: common-commits@hadoop.apache.org Message-Id: <284a383cf15c4f13b94648488071e1fe@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: YARN-3489. RMServerUtils.validateResourceRequests should only obtain queue info once. (Varun Saxena via wangda) Date: Fri, 29 May 2015 00:05:44 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/branch-2.7 7489b128d -> bb8350388 YARN-3489. RMServerUtils.validateResourceRequests should only obtain queue info once. (Varun Saxena via wangda) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bb835038 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bb835038 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bb835038 Branch: refs/heads/branch-2.7 Commit: bb8350388bc8933183d657f5cf3b78afefb94f68 Parents: 7489b12 Author: Wangda Tan Authored: Thu May 28 17:04:20 2015 -0700 Committer: Wangda Tan Committed: Thu May 28 17:04:20 2015 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 5 +-- .../server/resourcemanager/RMServerUtils.java | 10 +++++- .../scheduler/SchedulerUtils.java | 34 +++++++++++++++----- 3 files changed, 38 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb835038/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 409178e..e1af21d 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -17,6 +17,9 @@ Release 2.7.1 - UNRELEASED YARN-3723. Need to clearly document primaryFilter and otherInfo value type. (Zhijie Shen via xgong) + YARN-3489. RMServerUtils.validateResourceRequests should only obtain queue + info once. (Varun Saxena via wangda) + OPTIMIZATIONS YARN-3006. Improve the error message when attempting manual failover with @@ -86,7 +89,6 @@ Release 2.7.1 - UNRELEASED YARN-3493. RM fails to come up with error "Failed to load/recover state" when mem settings are changed. (Jian He via wangda) -<<<<<<< HEAD YARN-3626. On Windows localized resources are not moved to the front of the classpath when they should be. (Craig Welch via xgong) @@ -1131,7 +1133,6 @@ Release 2.6.0 - 2014-11-18 YARN-2406. Move RM recovery related proto to yarn_server_resourcemanager_recovery.proto. (Tsuyoshi Ozawa via jianhe) -<<<<<<< HEAD YARN-1506. Changed RMNode/SchedulerNode to update resource with event notification. (Junping Du via jianhe) http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb835038/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java index 4669a28..fd17153 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java @@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; import org.apache.hadoop.yarn.api.records.ResourceRequest; @@ -94,9 +95,16 @@ public class RMServerUtils { Resource maximumResource, String queueName, YarnScheduler scheduler, RMContext rmContext) throws InvalidResourceRequestException { + + QueueInfo queueInfo = null; + try { + queueInfo = scheduler.getQueueInfo(queueName, false, false); + } catch (IOException e) { + } + for (ResourceRequest resReq : ask) { SchedulerUtils.normalizeAndvalidateRequest(resReq, maximumResource, - queueName, scheduler, rmContext); + queueName, scheduler, rmContext, queueInfo); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb835038/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java index 45730be..071f293 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java @@ -212,28 +212,46 @@ public class SchedulerUtils { public static void normalizeAndValidateRequest(ResourceRequest resReq, Resource maximumResource, String queueName, YarnScheduler scheduler, - boolean isRecovery, RMContext rmContext) + boolean isRecovery, RMContext rmContext, QueueInfo queueInfo) throws InvalidResourceRequestException { - QueueInfo queueInfo = null; - try { - queueInfo = scheduler.getQueueInfo(queueName, false, false); - } catch (IOException e) { - // it is possible queue cannot get when queue mapping is set, just ignore - // the queueInfo here, and move forward + if (queueInfo == null) { + try { + queueInfo = scheduler.getQueueInfo(queueName, false, false); + } catch (IOException e) { + // it is possible queue cannot get when queue mapping is set, just ignore + // the queueInfo here, and move forward + } } SchedulerUtils.normalizeNodeLabelExpressionInRequest(resReq, queueInfo); if (!isRecovery) { validateResourceRequest(resReq, maximumResource, queueInfo, rmContext); } } + + public static void normalizeAndValidateRequest(ResourceRequest resReq, + Resource maximumResource, String queueName, YarnScheduler scheduler, + boolean isRecovery, RMContext rmContext) + throws InvalidResourceRequestException { + normalizeAndValidateRequest(resReq, maximumResource, queueName, scheduler, + isRecovery, rmContext, null); + } + + public static void normalizeAndvalidateRequest(ResourceRequest resReq, + Resource maximumResource, String queueName, YarnScheduler scheduler, + RMContext rmContext, QueueInfo queueInfo) + throws InvalidResourceRequestException { + normalizeAndValidateRequest(resReq, maximumResource, queueName, scheduler, + false, rmContext, queueInfo); + } + public static void normalizeAndvalidateRequest(ResourceRequest resReq, Resource maximumResource, String queueName, YarnScheduler scheduler, RMContext rmContext) throws InvalidResourceRequestException { normalizeAndValidateRequest(resReq, maximumResource, queueName, scheduler, - false, rmContext); + false, rmContext, null); } /**