Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 0273C200D09 for ; Tue, 12 Sep 2017 18:33:45 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 00D091609B4; Tue, 12 Sep 2017 16:33:45 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id EB5BB1609C7 for ; Tue, 12 Sep 2017 18:33:43 +0200 (CEST) Received: (qmail 54505 invoked by uid 500); 12 Sep 2017 16:33:40 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 54443 invoked by uid 99); 12 Sep 2017 16:33:40 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 12 Sep 2017 16:33:40 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 7FD00F5724; Tue, 12 Sep 2017 16:33:40 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: asuresh@apache.org To: common-commits@hadoop.apache.org Message-Id: <6a95d645f59044db9bebe48e348cd6d4@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: YARN-7173. Container update RM-NM communication fix for backward compatibility. (Arun Suresh via wangda) Date: Tue, 12 Sep 2017 16:33:40 +0000 (UTC) archived-at: Tue, 12 Sep 2017 16:33:45 -0000 Repository: hadoop Updated Branches: refs/heads/branch-3.0 13310f91a -> d2e381882 YARN-7173. Container update RM-NM communication fix for backward compatibility. (Arun Suresh via wangda) Change-Id: I1c39ed5c59dee739ba5044b61b3ef5ed203b79c1 (cherry picked from commit e74d1be04be47969943b0501a4f335b0b5188287) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d2e38188 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d2e38188 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d2e38188 Branch: refs/heads/branch-3.0 Commit: d2e3818823763f374a42247bb4dd2ddf0f2a1741 Parents: 13310f9 Author: Wangda Tan Authored: Mon Sep 11 20:46:41 2017 -0700 Committer: Arun Suresh Committed: Tue Sep 12 08:09:47 2017 -0700 ---------------------------------------------------------------------- .../protocolrecords/NodeHeartbeatResponse.java | 5 ++ .../impl/pb/NodeHeartbeatResponsePBImpl.java | 65 ++++++++++++++++++++ .../rmcontainer/RMContainerImpl.java | 4 +- .../resourcemanager/rmnode/RMNodeImpl.java | 20 +++++- .../rmnode/RMNodeUpdateContainerEvent.java | 9 +-- .../scheduler/SchedulerApplicationAttempt.java | 3 +- 6 files changed, 97 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d2e38188/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java index 2ebca57..05a9c72 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java @@ -113,4 +113,9 @@ public abstract class NodeHeartbeatResponse { public abstract void setContainerQueuingLimit( ContainerQueuingLimit containerQueuingLimit); + + public abstract List getContainersToDecrease(); + + public abstract void addAllContainersToDecrease( + Collection containersToDecrease); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d2e38188/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java index 11f5f61..bbd1294 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java @@ -80,6 +80,8 @@ public class NodeHeartbeatResponsePBImpl extends NodeHeartbeatResponse { private MasterKey nmTokenMasterKey = null; private ContainerQueuingLimit containerQueuingLimit = null; private List containersToUpdate = null; + // NOTE: This is required for backward compatibility. + private List containersToDecrease = null; private List containersToSignal = null; public NodeHeartbeatResponsePBImpl() { @@ -126,6 +128,9 @@ public class NodeHeartbeatResponsePBImpl extends NodeHeartbeatResponse { if (this.containersToUpdate != null) { addContainersToUpdateToProto(); } + if (this.containersToDecrease != null) { + addContainersToDecreaseToProto(); + } if (this.containersToSignal != null) { addContainersToSignalToProto(); } @@ -572,6 +577,66 @@ public class NodeHeartbeatResponsePBImpl extends NodeHeartbeatResponse { builder.addAllContainersToUpdate(iterable); } + private void initContainersToDecrease() { + if (this.containersToDecrease != null) { + return; + } + NodeHeartbeatResponseProtoOrBuilder p = viaProto ? proto : builder; + List list = p.getContainersToDecreaseList(); + this.containersToDecrease = new ArrayList<>(); + + for (ContainerProto c : list) { + this.containersToDecrease.add(convertFromProtoFormat(c)); + } + } + + @Override + public List getContainersToDecrease() { + initContainersToDecrease(); + return this.containersToDecrease; + } + + @Override + public void addAllContainersToDecrease( + final Collection containersToDecrease) { + if (containersToDecrease == null) { + return; + } + initContainersToDecrease(); + this.containersToDecrease.addAll(containersToDecrease); + } + + private void addContainersToDecreaseToProto() { + maybeInitBuilder(); + builder.clearContainersToDecrease(); + if (this.containersToDecrease == null) { + return; + } + + Iterable iterable = new + Iterable() { + @Override + public Iterator iterator() { + return new Iterator() { + private Iterator iter = containersToDecrease.iterator(); + @Override + public boolean hasNext() { + return iter.hasNext(); + } + @Override + public ContainerProto next() { + return convertToProtoFormat(iter.next()); + } + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + }; + } + }; + builder.addAllContainersToDecrease(iterable); + } + @Override public Map getSystemCredentialsForApps() { if (this.systemCredentials != null) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/d2e38188/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java index f49db7e..8aa5788 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java @@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerReport; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.ContainerUpdateType; import org.apache.hadoop.yarn.api.records.ExecutionType; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; @@ -641,7 +642,8 @@ public class RMContainerImpl implements RMContainer { new AllocationExpirationInfo(event.getContainerId())); container.eventHandler.handle(new RMNodeUpdateContainerEvent( container.nodeId, - Collections.singletonList(container.getContainer()))); + Collections.singletonMap(container.getContainer(), + ContainerUpdateType.DECREASE_RESOURCE))); } else if (Resources.fitsIn(nmContainerResource, rmContainerResource)) { // If nmContainerResource < rmContainerResource, this is caused by the // following sequence: http://git-wip-us.apache.org/repos/asf/hadoop/blob/d2e38188/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java index d270aa3..c547128 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java @@ -48,6 +48,7 @@ import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.ContainerUpdateType; import org.apache.hadoop.yarn.api.records.ExecutionType; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeState; @@ -173,7 +174,11 @@ public class RMNodeImpl implements RMNode, EventHandler { private final Map toBeUpdatedContainers = new HashMap<>(); - + + // NOTE: This is required for backward compatibility. + private final Map toBeDecreasedContainers = + new HashMap<>(); + private final Map nmReportedIncreasedContainers = new HashMap<>(); @@ -626,6 +631,10 @@ public class RMNodeImpl implements RMNode, EventHandler { try { response.addAllContainersToUpdate(toBeUpdatedContainers.values()); toBeUpdatedContainers.clear(); + + // NOTE: This is required for backward compatibility. + response.addAllContainersToDecrease(toBeDecreasedContainers.values()); + toBeDecreasedContainers.clear(); } finally { this.writeLock.unlock(); } @@ -1043,8 +1052,13 @@ public class RMNodeImpl implements RMNode, EventHandler { public void transition(RMNodeImpl rmNode, RMNodeEvent event) { RMNodeUpdateContainerEvent de = (RMNodeUpdateContainerEvent) event; - for (Container c : de.getToBeUpdatedContainers()) { - rmNode.toBeUpdatedContainers.put(c.getId(), c); + for (Map.Entry e : + de.getToBeUpdatedContainers().entrySet()) { + // NOTE: This is required for backward compatibility. + if (ContainerUpdateType.DECREASE_RESOURCE == e.getValue()) { + rmNode.toBeDecreasedContainers.put(e.getKey().getId(), e.getKey()); + } + rmNode.toBeUpdatedContainers.put(e.getKey().getId(), e.getKey()); } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d2e38188/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeUpdateContainerEvent.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeUpdateContainerEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeUpdateContainerEvent.java index 73af563..b8f8e73 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeUpdateContainerEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeUpdateContainerEvent.java @@ -19,8 +19,10 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmnode; import java.util.List; +import java.util.Map; import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerUpdateType; import org.apache.hadoop.yarn.api.records.NodeId; /** @@ -29,16 +31,15 @@ import org.apache.hadoop.yarn.api.records.NodeId; * */ public class RMNodeUpdateContainerEvent extends RMNodeEvent { - private List toBeUpdatedContainers; + private Map toBeUpdatedContainers; public RMNodeUpdateContainerEvent(NodeId nodeId, - List toBeUpdatedContainers) { + Map toBeUpdatedContainers) { super(nodeId, RMNodeEventType.UPDATE_CONTAINER); - this.toBeUpdatedContainers = toBeUpdatedContainers; } - public List getToBeUpdatedContainers() { + public Map getToBeUpdatedContainers() { return toBeUpdatedContainers; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d2e38188/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index 6a44cae..c807590 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -690,7 +690,8 @@ public class SchedulerApplicationAttempt implements SchedulableEntity { if (autoUpdate) { this.rmContext.getDispatcher().getEventHandler().handle( new RMNodeUpdateContainerEvent(rmContainer.getNodeId(), - Collections.singletonList(rmContainer.getContainer()))); + Collections.singletonMap( + rmContainer.getContainer(), updateType))); } else { rmContainer.handle(new RMContainerUpdatesAcquiredEvent( rmContainer.getContainerId(), --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org