Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 345DA200D5B for ; Tue, 28 Nov 2017 22:47:36 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 32CC7160C17; Tue, 28 Nov 2017 21:47:36 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 0525E160BE7 for ; Tue, 28 Nov 2017 22:47:34 +0100 (CET) Received: (qmail 89072 invoked by uid 500); 28 Nov 2017 21:47:22 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 86351 invoked by uid 99); 28 Nov 2017 21:47:21 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 28 Nov 2017 21:47:21 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 1A13DF6041; Tue, 28 Nov 2017 21:47:20 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: kkaranasos@apache.org To: common-commits@hadoop.apache.org Date: Tue, 28 Nov 2017 21:48:07 -0000 Message-Id: <584109c0131c41d7acc67921e78c07ee@git.apache.org> In-Reply-To: <7246f4abfe7d4e1b850d697f7cf2a998@git.apache.org> References: <7246f4abfe7d4e1b850d697f7cf2a998@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [49/50] [abbrv] hadoop git commit: YARN-6595. [API] Add Placement Constraints at the application level. (Arun Suresh via kkaranasos) archived-at: Tue, 28 Nov 2017 21:47:36 -0000 YARN-6595. [API] Add Placement Constraints at the application level. (Arun Suresh via kkaranasos) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6a0abf39 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6a0abf39 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6a0abf39 Branch: refs/heads/YARN-6592 Commit: 6a0abf39ff3b1db9ea686324f84e43f9ae598ae2 Parents: 3571634 Author: Konstantinos Karanasos Authored: Mon Nov 13 15:25:24 2017 -0800 Committer: Konstantinos Karanasos Committed: Tue Nov 28 13:46:29 2017 -0800 ---------------------------------------------------------------------- .../RegisterApplicationMasterRequest.java | 42 ++++- .../yarn/api/resource/PlacementConstraint.java | 156 +++++++++++++++++++ .../src/main/proto/yarn_protos.proto | 6 + .../src/main/proto/yarn_service_protos.proto | 1 + .../RegisterApplicationMasterRequestPBImpl.java | 106 ++++++++++++- .../hadoop/yarn/api/BasePBImplRecordsTest.java | 11 ++ 6 files changed, 313 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a0abf39/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java index 395e190..f2d537a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java @@ -18,11 +18,16 @@ package org.apache.hadoop.yarn.api.protocolrecords; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint; import org.apache.hadoop.yarn.util.Records; - /** * The request sent by the {@code ApplicationMaster} to {@code ResourceManager} * on registration. @@ -132,4 +137,39 @@ public abstract class RegisterApplicationMasterRequest { @Public @Stable public abstract void setTrackingUrl(String trackingUrl); + + /** + * Return all Placement Constraints specified at the Application level. The + * mapping is from a set of allocation tags to a + * PlacementConstraint associated with the tags, i.e., each + * {@link org.apache.hadoop.yarn.api.records.SchedulingRequest} that has those + * tags will be placed taking into account the corresponding constraint. + * + * @return A map of Placement Constraints. + */ + @Public + @Unstable + public Map, PlacementConstraint> getPlacementConstraints() { + return new HashMap<>(); + } + + /** + * Set Placement Constraints applicable to the + * {@link org.apache.hadoop.yarn.api.records.SchedulingRequest}s + * of this application. + * The mapping is from a set of allocation tags to a + * PlacementConstraint associated with the tags. + * For example: + * Map < + * <hb_regionserver> -> node_anti_affinity, + * <hb_regionserver, hb_master> -> rack_affinity, + * ... + * > + * @param placementConstraints Placement Constraint Mapping. + */ + @Public + @Unstable + public void setPlacementConstraints( + Map, PlacementConstraint> placementConstraints) { + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a0abf39/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java index f0e3982..b6e851a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java @@ -54,6 +54,26 @@ public class PlacementConstraint { return constraintExpr; } + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof PlacementConstraint)) { + return false; + } + + PlacementConstraint that = (PlacementConstraint) o; + + return getConstraintExpr() != null ? getConstraintExpr().equals(that + .getConstraintExpr()) : that.getConstraintExpr() == null; + } + + @Override + public int hashCode() { + return getConstraintExpr() != null ? getConstraintExpr().hashCode() : 0; + } + /** * Interface used to enable the elements of the constraint tree to be visited. */ @@ -174,6 +194,38 @@ public class PlacementConstraint { } @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof SingleConstraint)) { + return false; + } + + SingleConstraint that = (SingleConstraint) o; + + if (getMinCardinality() != that.getMinCardinality()) { + return false; + } + if (getMaxCardinality() != that.getMaxCardinality()) { + return false; + } + if (!getScope().equals(that.getScope())) { + return false; + } + return getTargetExpressions().equals(that.getTargetExpressions()); + } + + @Override + public int hashCode() { + int result = getScope().hashCode(); + result = 31 * result + getMinCardinality(); + result = 31 * result + getMaxCardinality(); + result = 31 * result + getTargetExpressions().hashCode(); + return result; + } + + @Override public T accept(Visitor visitor) { return visitor.visit(this); } @@ -332,6 +384,34 @@ public class PlacementConstraint { } @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof TargetConstraint)) { + return false; + } + + TargetConstraint that = (TargetConstraint) o; + + if (getOp() != that.getOp()) { + return false; + } + if (!getScope().equals(that.getScope())) { + return false; + } + return getTargetExpressions().equals(that.getTargetExpressions()); + } + + @Override + public int hashCode() { + int result = getOp().hashCode(); + result = 31 * result + getScope().hashCode(); + result = 31 * result + getTargetExpressions().hashCode(); + return result; + } + + @Override public T accept(Visitor visitor) { return visitor.visit(this); } @@ -388,6 +468,34 @@ public class PlacementConstraint { public T accept(Visitor visitor) { return visitor.visit(this); } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + CardinalityConstraint that = (CardinalityConstraint) o; + + if (minCardinality != that.minCardinality) { + return false; + } + if (maxCardinality != that.maxCardinality) { + return false; + } + return scope != null ? scope.equals(that.scope) : that.scope == null; + } + + @Override + public int hashCode() { + int result = scope != null ? scope.hashCode() : 0; + result = 31 * result + minCardinality; + result = 31 * result + maxCardinality; + return result; + } } /** @@ -406,6 +514,25 @@ public class PlacementConstraint { * @return the children of the composite constraint */ public abstract List getChildren(); + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + return getChildren() != null ? getChildren().equals( + ((CompositeConstraint)o).getChildren()) : + ((CompositeConstraint)o).getChildren() == null; + } + + @Override + public int hashCode() { + return getChildren() != null ? getChildren().hashCode() : 0; + } } /** @@ -563,5 +690,34 @@ public class PlacementConstraint { public T accept(Visitor visitor) { return visitor.visit(this); } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + TimedPlacementConstraint that = (TimedPlacementConstraint) o; + + if (schedulingDelay != that.schedulingDelay) { + return false; + } + if (constraint != null ? !constraint.equals(that.constraint) : + that.constraint != null) { + return false; + } + return delayUnit == that.delayUnit; + } + + @Override + public int hashCode() { + int result = constraint != null ? constraint.hashCode() : 0; + result = 31 * result + (int) (schedulingDelay ^ (schedulingDelay >>> 32)); + result = 31 * result + (delayUnit != null ? delayUnit.hashCode() : 0); + return result; + } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a0abf39/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 3b26992..baba0aa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -644,6 +644,12 @@ message CompositePlacementConstraintProto { repeated TimedPlacementConstraintProto timedChildConstraints = 3; } +// This associates a set of allocation tags to a Placement Constraint. +message PlacementConstraintMapEntryProto { + repeated string allocation_tags = 1; + optional PlacementConstraintProto placement_constraint = 2; +} + //////////////////////////////////////////////////////////////////////// ////// From reservation_protocol ///////////////////////////////////// //////////////////////////////////////////////////////////////////////// http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a0abf39/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto index 4e97c74..68e585d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto @@ -38,6 +38,7 @@ message RegisterApplicationMasterRequestProto { optional string host = 1; optional int32 rpc_port = 2; optional string tracking_url = 3; + repeated PlacementConstraintMapEntryProto placement_constraints = 4; } message RegisterApplicationMasterResponseProto { http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a0abf39/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterRequestPBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterRequestPBImpl.java index 037dfd9..64bee85 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterRequestPBImpl.java @@ -21,24 +21,41 @@ package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.pb.PlacementConstraintFromProtoConverter; +import org.apache.hadoop.yarn.api.pb.PlacementConstraintToProtoConverter; import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest; + +import org.apache.hadoop.yarn.api.resource.PlacementConstraint; +import org.apache.hadoop.yarn.proto.YarnProtos; import org.apache.hadoop.yarn.proto.YarnServiceProtos.RegisterApplicationMasterRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.RegisterApplicationMasterRequestProtoOrBuilder; import com.google.protobuf.TextFormat; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + @Private @Unstable -public class RegisterApplicationMasterRequestPBImpl extends RegisterApplicationMasterRequest { - RegisterApplicationMasterRequestProto proto = RegisterApplicationMasterRequestProto.getDefaultInstance(); - RegisterApplicationMasterRequestProto.Builder builder = null; +public class RegisterApplicationMasterRequestPBImpl + extends RegisterApplicationMasterRequest { + private RegisterApplicationMasterRequestProto proto = + RegisterApplicationMasterRequestProto.getDefaultInstance(); + private RegisterApplicationMasterRequestProto.Builder builder = null; + private Map, PlacementConstraint> placementConstraints = null; boolean viaProto = false; public RegisterApplicationMasterRequestPBImpl() { builder = RegisterApplicationMasterRequestProto.newBuilder(); } - public RegisterApplicationMasterRequestPBImpl(RegisterApplicationMasterRequestProto proto) { + public RegisterApplicationMasterRequestPBImpl( + RegisterApplicationMasterRequestProto proto) { this.proto = proto; viaProto = true; } @@ -71,6 +88,30 @@ public class RegisterApplicationMasterRequestPBImpl extends RegisterApplicationM } private void mergeLocalToBuilder() { + if (this.placementConstraints != null) { + addPlacementConstraintMap(); + } + } + + private void addPlacementConstraintMap() { + maybeInitBuilder(); + builder.clearPlacementConstraints(); + if (this.placementConstraints == null) { + return; + } + List protoList = + new ArrayList<>(); + for (Map.Entry, PlacementConstraint> entry : + this.placementConstraints.entrySet()) { + protoList.add( + YarnProtos.PlacementConstraintMapEntryProto.newBuilder() + .addAllAllocationTags(entry.getKey()) + .setPlacementConstraint( + new PlacementConstraintToProtoConverter( + entry.getValue()).convert()) + .build()); + } + builder.addAllPlacementConstraints(protoList); } private void mergeLocalToProto() { @@ -90,7 +131,8 @@ public class RegisterApplicationMasterRequestPBImpl extends RegisterApplicationM @Override public String getHost() { - RegisterApplicationMasterRequestProtoOrBuilder p = viaProto ? proto : builder; + RegisterApplicationMasterRequestProtoOrBuilder p = + viaProto ? proto : builder; return p.getHost(); } @@ -106,7 +148,8 @@ public class RegisterApplicationMasterRequestPBImpl extends RegisterApplicationM @Override public int getRpcPort() { - RegisterApplicationMasterRequestProtoOrBuilder p = viaProto ? proto : builder; + RegisterApplicationMasterRequestProtoOrBuilder p = + viaProto ? proto : builder; return p.getRpcPort(); } @@ -118,7 +161,8 @@ public class RegisterApplicationMasterRequestPBImpl extends RegisterApplicationM @Override public String getTrackingUrl() { - RegisterApplicationMasterRequestProtoOrBuilder p = viaProto ? proto : builder; + RegisterApplicationMasterRequestProtoOrBuilder p = + viaProto ? proto : builder; return p.getTrackingUrl(); } @@ -131,4 +175,50 @@ public class RegisterApplicationMasterRequestPBImpl extends RegisterApplicationM } builder.setTrackingUrl(url); } -} + + private void initPlacementConstraintMap() { + if (this.placementConstraints != null) { + return; + } + RegisterApplicationMasterRequestProtoOrBuilder p = + viaProto ? proto : builder; + List pcmList = + p.getPlacementConstraintsList(); + this.placementConstraints = new HashMap<>(); + for (YarnProtos.PlacementConstraintMapEntryProto e : pcmList) { + this.placementConstraints.put( + new HashSet<>(e.getAllocationTagsList()), + new PlacementConstraintFromProtoConverter( + e.getPlacementConstraint()).convert()); + } + } + + @Override + public Map, PlacementConstraint> getPlacementConstraints() { + initPlacementConstraintMap(); + return this.placementConstraints; + } + + @Override + public void setPlacementConstraints( + Map, PlacementConstraint> constraints) { + maybeInitBuilder(); + if (constraints == null) { + builder.clearPlacementConstraints(); + } else { + removeEmptyKeys(constraints); + } + this.placementConstraints = constraints; + } + + private void removeEmptyKeys( + Map, PlacementConstraint> constraintMap) { + Iterator> iter = constraintMap.keySet().iterator(); + while (iter.hasNext()) { + Set aTags = iter.next(); + if (aTags.size() == 0) { + iter.remove(); + } + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a0abf39/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java index 8694651..ebd66af 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java @@ -22,12 +22,19 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint; +import org.apache.hadoop.yarn.api.resource.PlacementConstraints; import org.junit.Assert; import java.lang.reflect.*; import java.nio.ByteBuffer; import java.util.*; +import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.NODE; +import static org.apache.hadoop.yarn.api.resource.PlacementConstraints + .PlacementTargets.allocationTag; +import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.targetIn; + /** * Generic helper class to validate protocol records. */ @@ -85,6 +92,10 @@ public class BasePBImplRecordsTest { ByteBuffer buff = ByteBuffer.allocate(4); rand.nextBytes(buff.array()); return buff; + } else if (type.equals(PlacementConstraint.class)) { + PlacementConstraint.AbstractConstraint sConstraintExpr = + targetIn(NODE, allocationTag("foo")); + ret = PlacementConstraints.build(sConstraintExpr); } } else if (type instanceof ParameterizedType) { ParameterizedType pt = (ParameterizedType)type; --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org