Return-Path: X-Original-To: apmail-hadoop-yarn-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-yarn-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id A254FD658 for ; Thu, 30 May 2013 01:58:01 +0000 (UTC) Received: (qmail 21064 invoked by uid 500); 30 May 2013 01:58:01 -0000 Delivered-To: apmail-hadoop-yarn-commits-archive@hadoop.apache.org Received: (qmail 21037 invoked by uid 500); 30 May 2013 01:58:01 -0000 Mailing-List: contact yarn-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: yarn-commits@hadoop.apache.org Delivered-To: mailing list yarn-commits@hadoop.apache.org Received: (qmail 21023 invoked by uid 99); 30 May 2013 01:58:01 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 30 May 2013 01:58:01 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 30 May 2013 01:58:00 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 74C7C23889ED; Thu, 30 May 2013 01:57:40 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1487705 - in /hadoop/common/branches/branch-2/hadoop-yarn-project: ./ hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecord... Date: Thu, 30 May 2013 01:57:40 -0000 To: yarn-commits@hadoop.apache.org From: vinodkv@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20130530015740.74C7C23889ED@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: vinodkv Date: Thu May 30 01:57:39 2013 New Revision: 1487705 URL: http://svn.apache.org/r1487705 Log: YARN-714. Added NMTokens to be sent to AMs as part of heart-beat response. Contributed by Omkar Vinit Joshi. svn merge --ignore-ancestry -c 1487704 ../../trunk/ Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1487705&r1=1487704&r2=1487705&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Thu May 30 01:57:39 2013 @@ -201,6 +201,9 @@ Release 2.0.5-beta - UNRELEASED YARN-711. Copied BuilderUtil methods in individual API records as BuilderUtils is going to be dismantled. (Jian He via vinodkv) + YARN-714. Added NMTokens to be sent to AMs as part of heart-beat response. + (Omkar Vinit Joshi via vinodkv) + OPTIMIZATIONS YARN-512. Log aggregation root directory check is more expensive than it Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java?rev=1487705&r1=1487704&r2=1487705&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java (original) +++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java Thu May 30 01:57:39 2013 @@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.Token; /** *

The response sent by the ResourceManager the @@ -176,5 +177,24 @@ public interface AllocateResponse { @Private @Unstable public void setPreemptionMessage(PreemptionMessage request); + + @Public + @Stable + public void setNMTokens(List nmTokens); + + /** + * Get the list of NMTokens required for communicating with NM. New NMTokens + * issued only if + * 1) AM is receiving first container on underlying NodeManager. + * OR + * 2) NMToken master key rolled over in ResourceManager and AM is getting new + * container on the same underlying NodeManager. + * AM will receive one NMToken per NM irrespective of the number of containers + * issued on same NM. AM is expected to store these tokens until issued a + * new token for the same NM. + */ + @Public + @Stable + public List getNMTokens(); } Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java?rev=1487705&r1=1487704&r2=1487705&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java (original) +++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java Thu May 30 01:57:39 2013 @@ -23,6 +23,7 @@ import java.util.ArrayList; import java.util.Iterator; import java.util.List; +import org.apache.hadoop.security.proto.SecurityProtos.TokenProto; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; import org.apache.hadoop.yarn.api.protocolrecords.PreemptionMessage; import org.apache.hadoop.yarn.api.records.Container; @@ -30,10 +31,12 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.ProtoBase; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ContainerStatusPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.NodeReportPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.TokenPBImpl; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerProto; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStatusProto; import org.apache.hadoop.yarn.proto.YarnProtos.NodeReportProto; @@ -52,6 +55,7 @@ public class AllocateResponsePBImpl exte Resource limit; private List allocatedContainers = null; + private List nmTokens = null; private List completedContainersStatuses = null; private List updatedNodes = null; @@ -81,6 +85,11 @@ public class AllocateResponsePBImpl exte getProtoIterable(this.allocatedContainers); builder.addAllAllocatedContainers(iterable); } + if (nmTokens != null) { + builder.clearNmTokens(); + Iterable iterable = getTokenProtoIterable(nmTokens); + builder.addAllNmTokens(iterable); + } if (this.completedContainersStatuses != null) { builder.clearCompletedContainerStatuses(); Iterable iterable = @@ -211,6 +220,24 @@ public class AllocateResponsePBImpl exte } @Override + public synchronized void setNMTokens(List nmTokens) { + if (nmTokens == null || nmTokens.isEmpty()) { + this.nmTokens.clear(); + builder.clearNmTokens(); + return; + } + // Implementing it as an append rather than set for consistency + initLocalNewNMTokenList(); + this.nmTokens.addAll(nmTokens); + } + + @Override + public synchronized List getNMTokens() { + initLocalNewNMTokenList(); + return nmTokens; + } + + @Override public synchronized int getNumClusterNodes() { AllocateResponseProtoOrBuilder p = viaProto ? proto : builder; return p.getNumClusterNodes(); @@ -274,6 +301,18 @@ public class AllocateResponsePBImpl exte } } + private synchronized void initLocalNewNMTokenList() { + if (nmTokens != null) { + return; + } + AllocateResponseProtoOrBuilder p = viaProto ? proto : builder; + List list = p.getNmTokensList(); + nmTokens = new ArrayList(); + for (TokenProto t : list) { + nmTokens.add(convertFromProtoFormat(t)); + } + } + private synchronized Iterable getProtoIterable( final List newContainersList) { maybeInitBuilder(); @@ -305,6 +344,35 @@ public class AllocateResponsePBImpl exte }; } + private synchronized Iterable getTokenProtoIterable( + final List nmTokenList) { + maybeInitBuilder(); + return new Iterable() { + @Override + public synchronized Iterator iterator() { + return new Iterator() { + + Iterator iter = nmTokenList.iterator(); + + @Override + public boolean hasNext() { + return iter.hasNext(); + } + + @Override + public TokenProto next() { + return convertToProtoFormat(iter.next()); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + }; + } + }; + } + private synchronized Iterable getContainerStatusProtoIterable( final List newContainersList) { @@ -427,4 +495,12 @@ public class AllocateResponsePBImpl exte private synchronized PreemptionMessageProto convertToProtoFormat(PreemptionMessage r) { return ((PreemptionMessagePBImpl)r).getProto(); } + + private synchronized TokenProto convertToProtoFormat(Token token) { + return ((TokenPBImpl)token).getProto(); + } + + private synchronized Token convertFromProtoFormat(TokenProto proto) { + return new TokenPBImpl(proto); + } } Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto?rev=1487705&r1=1487704&r2=1487705&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto (original) +++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto Thu May 30 01:57:39 2013 @@ -67,6 +67,7 @@ message AllocateResponseProto { repeated NodeReportProto updated_nodes = 6; optional int32 num_cluster_nodes = 7; optional PreemptionMessageProto preempt = 8; + repeated hadoop.common.TokenProto nm_tokens = 9; } message PreemptionMessageProto {