Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id E83AA2009F4 for ; Thu, 26 May 2016 23:58:28 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id E6A08160A18; Thu, 26 May 2016 21:58:28 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 14817160A17 for ; Thu, 26 May 2016 23:58:27 +0200 (CEST) Received: (qmail 75465 invoked by uid 500); 26 May 2016 21:58:27 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 75456 invoked by uid 99); 26 May 2016 21:58:27 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 26 May 2016 21:58:27 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 1B1CBDFB38; Thu, 26 May 2016 21:58:27 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: asuresh@apache.org To: common-commits@hadoop.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: YARN-5162. Fix Exceptions thrown during in registerAM call when Distributed Scheduling is Enabled (Hitesh Sharma via asuresh) Date: Thu, 26 May 2016 21:58:27 +0000 (UTC) archived-at: Thu, 26 May 2016 21:58:29 -0000 Repository: hadoop Updated Branches: refs/heads/branch-2 b4c8729cf -> 44cbf5b7f YARN-5162. Fix Exceptions thrown during in registerAM call when Distributed Scheduling is Enabled (Hitesh Sharma via asuresh) (cherry picked from commit 5b41b288d01b0124664ddf6a3d6b545058bcfe6f) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/44cbf5b7 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/44cbf5b7 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/44cbf5b7 Branch: refs/heads/branch-2 Commit: 44cbf5b7f5097432729eb2b54379c0da29179738 Parents: b4c8729 Author: Arun Suresh Authored: Thu May 26 13:22:33 2016 -0700 Committer: Arun Suresh Committed: Thu May 26 14:58:15 2016 -0700 ---------------------------------------------------------------------- .../yarn/security/SchedulerSecurityInfo.java | 2 +- .../api/DistributedSchedulerProtocolPB.java | 10 +++++----- .../pb/DistSchedRegisterResponsePBImpl.java | 6 +++++- .../proto/distributed_scheduler_protocol.proto | 1 + .../TestDistributedSchedulingService.java | 20 ++++++++++++++++++++ 5 files changed, 32 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/44cbf5b7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/SchedulerSecurityInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/SchedulerSecurityInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/SchedulerSecurityInfo.java index bfc34d3..cf25273 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/SchedulerSecurityInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/SchedulerSecurityInfo.java @@ -41,7 +41,7 @@ public class SchedulerSecurityInfo extends SecurityInfo { @Override public TokenInfo getTokenInfo(Class protocol, Configuration conf) { - if (!protocol.equals(ApplicationMasterProtocolPB.class)) { + if (!ApplicationMasterProtocolPB.class.isAssignableFrom(protocol)) { return null; } return new TokenInfo() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/44cbf5b7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/DistributedSchedulerProtocolPB.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/DistributedSchedulerProtocolPB.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/DistributedSchedulerProtocolPB.java index 413b9c9..ce7911c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/DistributedSchedulerProtocolPB.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/DistributedSchedulerProtocolPB.java @@ -21,16 +21,16 @@ package org.apache.hadoop.yarn.server.api; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.ipc.ProtocolInfo; +import org.apache.hadoop.yarn.api.ApplicationMasterProtocolPB; import org.apache.hadoop.yarn.proto.ApplicationMasterProtocol.ApplicationMasterProtocolService; - - -import org.apache.hadoop.yarn.proto.DistributedSchedulerProtocol; +import org.apache.hadoop.yarn.proto.DistributedSchedulerProtocol.DistributedSchedulerProtocolService; @Private @Unstable @ProtocolInfo(protocolName = "org.apache.hadoop.yarn.server.api.DistributedSchedulerProtocolPB", protocolVersion = 1) public interface DistributedSchedulerProtocolPB extends - DistributedSchedulerProtocol.DistributedSchedulerProtocolService.BlockingInterface, - ApplicationMasterProtocolService.BlockingInterface { + DistributedSchedulerProtocolService.BlockingInterface, + ApplicationMasterProtocolService.BlockingInterface, + ApplicationMasterProtocolPB { } http://git-wip-us.apache.org/repos/asf/hadoop/blob/44cbf5b7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/DistSchedRegisterResponsePBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/DistSchedRegisterResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/DistSchedRegisterResponsePBImpl.java index 0322c70..eec62da 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/DistSchedRegisterResponsePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/DistSchedRegisterResponsePBImpl.java @@ -94,9 +94,13 @@ public class DistSchedRegisterResponsePBImpl extends DistSchedRegisterResponse { ProtoUtils.convertToProtoFormat(this.maxAllocatableCapability)); } if (this.minAllocatableCapability != null) { - builder.setMaxAllocCapability( + builder.setMinAllocCapability( ProtoUtils.convertToProtoFormat(this.minAllocatableCapability)); } + if (this.incrAllocatableCapability != null) { + builder.setIncrAllocCapability( + ProtoUtils.convertToProtoFormat(this.incrAllocatableCapability)); + } if (this.registerApplicationMasterResponse != null) { builder.setRegisterResponse( ((RegisterApplicationMasterResponsePBImpl) http://git-wip-us.apache.org/repos/asf/hadoop/blob/44cbf5b7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/distributed_scheduler_protocol.proto ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/distributed_scheduler_protocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/distributed_scheduler_protocol.proto index 7e3a77f..b94656c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/distributed_scheduler_protocol.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/distributed_scheduler_protocol.proto @@ -34,5 +34,6 @@ import "yarn_server_common_service_protos.proto"; service DistributedSchedulerProtocolService { rpc registerApplicationMasterForDistributedScheduling (RegisterApplicationMasterRequestProto) returns (DistSchedRegisterResponseProto); + rpc finishApplicationMaster (FinishApplicationMasterRequestProto) returns (FinishApplicationMasterResponseProto); rpc allocateForDistributedScheduling (AllocateRequestProto) returns (DistSchedAllocateResponseProto); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/44cbf5b7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestDistributedSchedulingService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestDistributedSchedulingService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestDistributedSchedulingService.java index 49bf4d0..47563d5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestDistributedSchedulingService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestDistributedSchedulingService.java @@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb .RegisterApplicationMasterRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb .RegisterApplicationMasterResponsePBImpl; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.api.DistributedSchedulerProtocolPB; import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; @@ -132,6 +133,9 @@ public class TestDistributedSchedulingService { DistSchedRegisterResponse resp = factory.newRecordInstance( DistSchedRegisterResponse.class); resp.setContainerIdStart(54321l); + resp.setMaxAllocatableCapabilty(Resource.newInstance(4096, 4)); + resp.setMinAllocatableCapabilty(Resource.newInstance(1024, 1)); + resp.setIncrAllocatableCapabilty(Resource.newInstance(2048, 2)); return resp; } @@ -194,6 +198,13 @@ public class TestDistributedSchedulingService { .newRecordInstance(RegisterApplicationMasterRequest.class)) .getProto())); Assert.assertEquals(54321l, dsRegResp.getContainerIdStart()); + Assert.assertEquals(4, + dsRegResp.getMaxAllocatableCapabilty().getVirtualCores()); + Assert.assertEquals(1024, + dsRegResp.getMinAllocatableCapabilty().getMemory()); + Assert.assertEquals(2, + dsRegResp.getIncrAllocatableCapabilty().getVirtualCores()); + DistSchedAllocateResponse dsAllocResp = new DistSchedAllocateResponsePBImpl( dsProxy.allocateForDistributedScheduling(null, @@ -201,5 +212,14 @@ public class TestDistributedSchedulingService { .newRecordInstance(AllocateRequest.class)).getProto())); Assert.assertEquals( "h1", dsAllocResp.getNodesForScheduling().get(0).getHost()); + + FinishApplicationMasterResponse dsfinishResp = + new FinishApplicationMasterResponsePBImpl( + dsProxy.finishApplicationMaster(null, + ((FinishApplicationMasterRequestPBImpl) factory + .newRecordInstance(FinishApplicationMasterRequest.class)) + .getProto())); + Assert.assertEquals( + false, dsfinishResp.getIsUnregistered()); } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org