From common-commits-return-87971-archive-asf-public=cust-asf.ponee.io@hadoop.apache.org Mon Sep 17 06:38:04 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 25FA6180627 for ; Mon, 17 Sep 2018 06:38:03 +0200 (CEST) Received: (qmail 4581 invoked by uid 500); 17 Sep 2018 04:38:02 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 4572 invoked by uid 99); 17 Sep 2018 04:38:02 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 17 Sep 2018 04:38:02 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 34EB1E08D7; Mon, 17 Sep 2018 04:38:02 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: sunilg@apache.org To: common-commits@hadoop.apache.org Message-Id: <2c5417b76d80480a81c6f4f8c330e07f@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: YARN-8715. Make allocation tags in the placement spec optional for node-attributes. Contributed by Weiwei Yang. Date: Mon, 17 Sep 2018 04:38:02 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/trunk 95231f174 -> 33d8327cf YARN-8715. Make allocation tags in the placement spec optional for node-attributes. Contributed by Weiwei Yang. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/33d8327c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/33d8327c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/33d8327c Branch: refs/heads/trunk Commit: 33d8327cffdc483b538aec3022fd8730b85babdb Parents: 95231f1 Author: Sunil G Authored: Mon Sep 17 10:07:45 2018 +0530 Committer: Sunil G Committed: Mon Sep 17 10:07:45 2018 +0530 ---------------------------------------------------------------------- .../constraint/PlacementConstraintParser.java | 44 ++++++++++++++++++-- .../resource/TestPlacementConstraintParser.java | 22 ++++++++++ .../distributedshell/ApplicationMaster.java | 20 +++++++-- 3 files changed, 78 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/33d8327c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/constraint/PlacementConstraintParser.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/constraint/PlacementConstraintParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/constraint/PlacementConstraintParser.java index 93fd706..de9419a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/constraint/PlacementConstraintParser.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/constraint/PlacementConstraintParser.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.yarn.util.constraint; +import com.google.common.base.Strings; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.yarn.api.records.NodeAttributeOpCode; @@ -589,6 +590,14 @@ public final class PlacementConstraintParser { this.num = number; } + public static SourceTags emptySourceTags() { + return new SourceTags("", 0); + } + + public boolean isEmpty() { + return Strings.isNullOrEmpty(tag) && num == 0; + } + public String getTag() { return this.tag; } @@ -692,20 +701,47 @@ public final class PlacementConstraintParser { // foo=4,Pn String[] splitted = specStr.split( String.valueOf(EXPRESSION_VAL_DELIM), 2); - if (splitted.length != 2) { + final SourceTags st; + final String exprs; + if (splitted.length == 1) { + // source tags not specified + exprs = splitted[0]; + st = SourceTags.emptySourceTags(); + } else if (splitted.length == 2) { + exprs = splitted[1]; + String tagAlloc = splitted[0]; + st = SourceTags.parseFrom(tagAlloc); + } else { throw new PlacementConstraintParseException( "Unexpected placement constraint expression " + specStr); } - String tagAlloc = splitted[0]; - SourceTags st = SourceTags.parseFrom(tagAlloc); - String exprs = splitted[1]; AbstractConstraint constraint = PlacementConstraintParser.parseExpression(exprs); result.put(st, constraint.build()); } + // Validation + Set sourceTagSet = result.keySet(); + if (sourceTagSet.stream() + .filter(sourceTags -> sourceTags.isEmpty()) + .findAny() + .isPresent()) { + // Source tags, e.g foo=3, is optional for a node-attribute constraint, + // but when source tags is absent, the parser only accept single + // constraint expression to avoid ambiguous semantic. This is because + // DS AM is requesting number of containers per the number specified + // in the source tags, we do overwrite when there is no source tags + // with num_containers argument from commandline. If that is partially + // missed in the constraints, we don't know if it is ought to + // overwritten or not. + if (result.size() != 1) { + throw new PlacementConstraintParseException( + "Source allocation tags is required for a multi placement" + + " constraint expression."); + } + } return result; } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/33d8327c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintParser.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintParser.java index 9806ba4..91e4fdb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintParser.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintParser.java @@ -501,5 +501,27 @@ public class TestPlacementConstraintParser { actualPc2 = valueIt.next(); Assert.assertEquals(expectedPc1, actualPc1.getConstraintExpr()); Assert.assertEquals(expectedPc2, actualPc2.getConstraintExpr()); + + // A single node attribute constraint w/o source tags + result = PlacementConstraintParser + .parsePlacementSpec("rm.yarn.io/foo=true"); + Assert.assertEquals(1, result.size()); + target = PlacementTargets.nodeAttribute("rm.yarn.io/foo", "true"); + expectedPc1 = targetNodeAttribute("node", NodeAttributeOpCode.EQ, target); + + SourceTags actualSourceTags = result.keySet().iterator().next(); + Assert.assertTrue(actualSourceTags.isEmpty()); + actualPc1 = result.values().iterator().next(); + Assert.assertEquals(expectedPc1, actualPc1.getConstraintExpr()); + + // If source tags is not specified for a node-attribute constraint, + // then this expression must be single constraint expression. + try { + PlacementConstraintParser + .parsePlacementSpec("rm.yarn.io/foo=true:xyz=1,notin,node,xyz"); + Assert.fail("Expected a failure!"); + } catch (Exception e) { + Assert.assertTrue(e instanceof PlacementConstraintParseException); + } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/33d8327c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java index 09a796e..a04f57b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java @@ -47,6 +47,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.Arrays; import java.util.concurrent.atomic.AtomicLong; +import com.google.common.base.Strings; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.GnuParser; import org.apache.commons.cli.HelpFormatter; @@ -527,7 +528,9 @@ public class ApplicationMaster { LOG.info("Placement Spec received [{}]", decodedSpec); this.numTotalContainers = 0; - parsePlacementSpecs(decodedSpec); + int globalNumOfContainers = Integer + .parseInt(cliParser.getOptionValue("num_containers", "0")); + parsePlacementSpecs(decodedSpec, globalNumOfContainers); LOG.info("Total num containers requested [{}]", numTotalContainers); if (numTotalContainers == 0) { @@ -698,11 +701,19 @@ public class ApplicationMaster { return true; } - private void parsePlacementSpecs(String decodedSpec) { + private void parsePlacementSpecs(String decodedSpec, + int globalNumOfContainers) { Map pSpecs = PlacementSpec.parse(decodedSpec); this.placementSpecs = new HashMap<>(); for (PlacementSpec pSpec : pSpecs.values()) { + // Use global num of containers when the spec doesn't specify + // source tags. This is allowed when using node-attribute constraints. + if (Strings.isNullOrEmpty(pSpec.sourceTag) + && pSpec.getNumContainers() == 0 + && globalNumOfContainers > 0) { + pSpec.setNumContainers(globalNumOfContainers); + } this.numTotalContainers += pSpec.getNumContainers(); this.placementSpecs.put(pSpec.sourceTag, pSpec); } @@ -799,8 +810,9 @@ public class ApplicationMaster { placementConstraintMap = new HashMap<>(); for (PlacementSpec spec : this.placementSpecs.values()) { if (spec.constraint != null) { - placementConstraintMap.put( - Collections.singleton(spec.sourceTag), spec.constraint); + Set allocationTags = Strings.isNullOrEmpty(spec.sourceTag) ? + Collections.emptySet() : Collections.singleton(spec.sourceTag); + placementConstraintMap.put(allocationTags, spec.constraint); } } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org