Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 4034718D6C for ; Tue, 8 Mar 2016 22:30:06 +0000 (UTC) Received: (qmail 21523 invoked by uid 500); 8 Mar 2016 22:30:05 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 21461 invoked by uid 500); 8 Mar 2016 22:30:05 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 21452 invoked by uid 99); 8 Mar 2016 22:30:05 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 08 Mar 2016 22:30:05 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id EA706DFB8A; Tue, 8 Mar 2016 22:30:04 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: wangda@apache.org To: common-commits@hadoop.apache.org Message-Id: <3cbe0209d59248919a37948565c92fb3@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: YARN-4465. SchedulerUtils#validateRequest for Label check should happen only when nodelabel enabled. (Bibin A Chundatt via wangda) Date: Tue, 8 Mar 2016 22:30:04 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/branch-2 7e29d732e -> f7b38a7fb YARN-4465. SchedulerUtils#validateRequest for Label check should happen only when nodelabel enabled. (Bibin A Chundatt via wangda) (cherry picked from commit 0233d4e0ee9947a95c018b1539310fc0bff6c44e) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f7b38a7f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f7b38a7f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f7b38a7f Branch: refs/heads/branch-2 Commit: f7b38a7fb8bb84b9b251c06c24c32d81c9e36730 Parents: 7e29d73 Author: Wangda Tan Authored: Tue Mar 8 14:27:03 2016 -0800 Committer: Wangda Tan Committed: Tue Mar 8 14:28:26 2016 -0800 ---------------------------------------------------------------------- .../scheduler/SchedulerUtils.java | 56 +++++++++++++------- .../scheduler/TestSchedulerUtils.java | 34 +++++++++++- 2 files changed, 70 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7b38a7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java index a80e921..b460964 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java @@ -24,6 +24,7 @@ import java.util.Set; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ContainerExitStatus; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; @@ -32,6 +33,7 @@ import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.InvalidLabelResourceRequestException; import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException; import org.apache.hadoop.yarn.factories.RecordFactory; @@ -222,6 +224,17 @@ public class SchedulerUtils { Resource maximumResource, String queueName, YarnScheduler scheduler, boolean isRecovery, RMContext rmContext, QueueInfo queueInfo) throws InvalidResourceRequestException { + Configuration conf = rmContext.getYarnConfiguration(); + // If Node label is not enabled throw exception + if (null != conf && !YarnConfiguration.areNodeLabelsEnabled(conf)) { + String labelExp = resReq.getNodeLabelExpression(); + if (!(RMNodeLabelsManager.NO_LABEL.equals(labelExp) + || null == labelExp)) { + throw new InvalidLabelResourceRequestException( + "Invalid resource request, node label not enabled " + + "but request contains label expression"); + } + } if (null == queueInfo) { try { queueInfo = scheduler.getQueueInfo(queueName, false, false); @@ -283,8 +296,8 @@ public class SchedulerUtils { // we don't allow specify label expression other than resourceName=ANY now if (!ResourceRequest.ANY.equals(resReq.getResourceName()) && labelExp != null && !labelExp.trim().isEmpty()) { - throw new InvalidResourceRequestException( - "Invailid resource request, queue=" + queueInfo.getQueueName() + throw new InvalidLabelResourceRequestException( + "Invalid resource request, queue=" + queueInfo.getQueueName() + " specified node label expression in a " + "resource request has resource name = " + resReq.getResourceName()); @@ -303,15 +316,28 @@ public class SchedulerUtils { if (!checkQueueLabelExpression(queueInfo.getAccessibleNodeLabels(), labelExp, rmContext)) { throw new InvalidLabelResourceRequestException( - "Invalid resource request" - + ", queue=" - + queueInfo.getQueueName() - + " doesn't have permission to access all labels " - + "in resource request. labelExpression of resource request=" - + labelExp - + ". Queue labels=" - + (queueInfo.getAccessibleNodeLabels() == null ? "" : StringUtils.join(queueInfo - .getAccessibleNodeLabels().iterator(), ','))); + "Invalid resource request" + ", queue=" + queueInfo.getQueueName() + + " doesn't have permission to access all labels " + + "in resource request. labelExpression of resource request=" + + labelExp + ". Queue labels=" + + (queueInfo.getAccessibleNodeLabels() == null ? "" + : StringUtils.join( + queueInfo.getAccessibleNodeLabels().iterator(), ','))); + } else { + checkQueueLabelInLabelManager(labelExp, rmContext); + } + } + } + + private static void checkQueueLabelInLabelManager(String labelExpression, + RMContext rmContext) throws InvalidLabelResourceRequestException { + // check node label manager contains this label + if (null != rmContext) { + RMNodeLabelsManager nlm = rmContext.getNodeLabelManager(); + if (nlm != null && !nlm.containsNodeLabel(labelExpression)) { + throw new InvalidLabelResourceRequestException( + "Invalid label resource request, cluster do not contain " + + ", label= " + labelExpression); } } } @@ -338,14 +364,6 @@ public class SchedulerUtils { return false; } } - - // check node label manager contains this label - if (null != rmContext) { - RMNodeLabelsManager nlm = rmContext.getNodeLabelManager(); - if (nlm != null && !nlm.containsNodeLabel(str)) { - return false; - } - } } } return true; http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7b38a7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java index 0e84d38..3208819 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java @@ -95,7 +95,8 @@ public class TestSchedulerUtils { private static final Log LOG = LogFactory.getLog(TestSchedulerUtils.class); private RMContext rmContext = getMockRMContext(); - + private static YarnConfiguration conf = new YarnConfiguration(); + @Test (timeout = 30000) public void testNormalizeRequest() { ResourceCalculator resourceCalculator = new DefaultResourceCalculator(); @@ -464,6 +465,34 @@ public class TestSchedulerUtils { rmContext.getNodeLabelManager().removeFromClusterNodeLabels( Arrays.asList("x")); } + try { + Resource resource = Resources.createResource(0, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); + ResourceRequest resReq1 = BuilderUtils + .newResourceRequest(mock(Priority.class), "*", resource, 1, "x"); + SchedulerUtils.normalizeAndvalidateRequest(resReq1, maxResource, "queue", + scheduler, rmContext); + fail("Should fail"); + } catch (InvalidResourceRequestException e) { + assertEquals("Invalid label resource request, cluster do not contain , " + + "label= x", e.getMessage()); + } + + try { + rmContext.getYarnConfiguration() + .set(YarnConfiguration.NODE_LABELS_ENABLED, "false"); + Resource resource = Resources.createResource(0, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); + ResourceRequest resReq1 = BuilderUtils + .newResourceRequest(mock(Priority.class), "*", resource, 1, "x"); + SchedulerUtils.normalizeAndvalidateRequest(resReq1, maxResource, "queue", + scheduler, rmContext); + Assert.assertEquals(RMNodeLabelsManager.NO_LABEL, + resReq1.getNodeLabelExpression()); + } catch (InvalidResourceRequestException e) { + assertEquals("Invalid resource request, node label not enabled but " + + "request contains label expression", e.getMessage()); + } } @Test (timeout = 30000) @@ -773,6 +802,9 @@ public class TestSchedulerUtils { RMContext rmContext = mock(RMContext.class); RMNodeLabelsManager nlm = new NullRMNodeLabelsManager(); nlm.init(new Configuration(false)); + when(rmContext.getYarnConfiguration()).thenReturn(conf); + rmContext.getYarnConfiguration().set(YarnConfiguration.NODE_LABELS_ENABLED, + "true"); when(rmContext.getNodeLabelManager()).thenReturn(nlm); return rmContext; }