Return-Path: X-Original-To: apmail-hadoop-yarn-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-yarn-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 55C5E1069A for ; Fri, 14 Jun 2013 00:22:12 +0000 (UTC) Received: (qmail 22346 invoked by uid 500); 14 Jun 2013 00:22:12 -0000 Delivered-To: apmail-hadoop-yarn-commits-archive@hadoop.apache.org Received: (qmail 22322 invoked by uid 500); 14 Jun 2013 00:22:12 -0000 Mailing-List: contact yarn-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: yarn-commits@hadoop.apache.org Delivered-To: mailing list yarn-commits@hadoop.apache.org Received: (qmail 22314 invoked by uid 99); 14 Jun 2013 00:22:12 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 14 Jun 2013 00:22:12 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 14 Jun 2013 00:22:00 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 8CA852388980; Fri, 14 Jun 2013 00:21:38 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1492915 [1/2] - in /hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project: ./ hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ hadoop-... Date: Fri, 14 Jun 2013 00:21:36 -0000 To: yarn-commits@hadoop.apache.org From: vinodkv@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20130614002138.8CA852388980@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: vinodkv Date: Fri Jun 14 00:21:34 2013 New Revision: 1492915 URL: http://svn.apache.org/r1492915 Log: YARN-692. Creating NMToken master key on RM and sharing it with NM as a part of RM-NM heartbeat. Contributed by Omkar Vinit Joshi. svn merge --ignore-ancestry -c 1492907 ../../trunk/ Added: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/NMTokenIdentifier.java - copied unchanged from r1492907, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/NMTokenIdentifier.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseNMTokenSecretManager.java - copied unchanged from r1492907, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseNMTokenSecretManager.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/MasterKeyData.java - copied unchanged from r1492907, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/MasterKeyData.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMTokenSecretManagerInNM.java - copied unchanged from r1492907, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMTokenSecretManagerInNM.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/NMTokenSecretManagerInRM.java - copied unchanged from r1492907, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/NMTokenSecretManagerInRM.java Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatRequest.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerResponse.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerResponsePBImpl.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/api/protocolrecords/TestRegisterNodeManagerResponse.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMContainerTokenSecretManager.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/LocalRMInterface.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/MockNodeStatusUpdater.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestRMNMSecretKeys.java Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt?rev=1492915&r1=1492914&r2=1492915&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt Fri Jun 14 00:21:34 2013 @@ -287,6 +287,9 @@ Release 2.1.0-beta - UNRELEASED YARN-773. Moved YarnRuntimeException from package api.yarn to api.yarn.exceptions. (Jian He via vinodkv) + YARN-692. Creating NMToken master key on RM and sharing it with NM as a part + of RM-NM heartbeat. (Omkar Vinit Joshi via vinodkv) + OPTIMIZATIONS YARN-512. Log aggregation root directory check is more expensive than it Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1492915&r1=1492914&r2=1492915&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Fri Jun 14 00:21:34 2013 @@ -286,6 +286,11 @@ public class YarnConfiguration extends C public static final long DEFAULT_RM_CONTAINER_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = 24 * 60 * 60; + public static final String RM_NMTOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = + RM_PREFIX + "nm-tokens.master-key-rolling-interval-secs"; + + public static final long DEFAULT_RM_NMTOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = + 24 * 60 * 60; //////////////////////////////// // Node Manager Configs //////////////////////////////// Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatRequest.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatRequest.java?rev=1492915&r1=1492914&r2=1492915&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatRequest.java (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatRequest.java Fri Jun 14 00:21:34 2013 @@ -26,6 +26,9 @@ public interface NodeHeartbeatRequest { NodeStatus getNodeStatus(); void setNodeStatus(NodeStatus status); - MasterKey getLastKnownMasterKey(); - void setLastKnownMasterKey(MasterKey secretKey); + MasterKey getLastKnownContainerTokenMasterKey(); + void setLastKnownContainerTokenMasterKey(MasterKey secretKey); + + MasterKey getLastKnownNMTokenMasterKey(); + void setLastKnownNMTokenMasterKey(MasterKey secretKey); } Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java?rev=1492915&r1=1492914&r2=1492915&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java Fri Jun 14 00:21:34 2013 @@ -36,8 +36,11 @@ public interface NodeHeartbeatResponse { void setResponseId(int responseId); void setNodeAction(NodeAction action); - MasterKey getMasterKey(); - void setMasterKey(MasterKey secretKey); + MasterKey getContainerTokenMasterKey(); + void setContainerTokenMasterKey(MasterKey secretKey); + + MasterKey getNMTokenMasterKey(); + void setNMTokenMasterKey(MasterKey secretKey); void addAllContainersToCleanup(List containers); Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerResponse.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerResponse.java?rev=1492915&r1=1492914&r2=1492915&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerResponse.java (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerResponse.java Fri Jun 14 00:21:34 2013 @@ -22,9 +22,13 @@ import org.apache.hadoop.yarn.server.api import org.apache.hadoop.yarn.server.api.records.NodeAction; public interface RegisterNodeManagerResponse { - MasterKey getMasterKey(); + MasterKey getContainerTokenMasterKey(); - void setMasterKey(MasterKey secretKey); + void setContainerTokenMasterKey(MasterKey secretKey); + + MasterKey getNMTokenMasterKey(); + + void setNMTokenMasterKey(MasterKey secretKey); NodeAction getNodeAction(); Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java?rev=1492915&r1=1492914&r2=1492915&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java Fri Jun 14 00:21:34 2013 @@ -36,7 +36,8 @@ public class NodeHeartbeatRequestPBImpl boolean viaProto = false; private NodeStatus nodeStatus = null; - private MasterKey lastKnownMasterKey = null; + private MasterKey lastKnownContainerTokenMasterKey = null; + private MasterKey lastKnownNMTokenMasterKey = null; public NodeHeartbeatRequestPBImpl() { builder = NodeHeartbeatRequestProto.newBuilder(); @@ -58,9 +59,13 @@ public class NodeHeartbeatRequestPBImpl if (this.nodeStatus != null) { builder.setNodeStatus(convertToProtoFormat(this.nodeStatus)); } - if (this.lastKnownMasterKey != null) { - builder - .setLastKnownMasterKey(convertToProtoFormat(this.lastKnownMasterKey)); + if (this.lastKnownContainerTokenMasterKey != null) { + builder.setLastKnownContainerTokenMasterKey( + convertToProtoFormat(this.lastKnownContainerTokenMasterKey)); + } + if (this.lastKnownNMTokenMasterKey != null) { + builder.setLastKnownNmTokenMasterKey( + convertToProtoFormat(this.lastKnownNMTokenMasterKey)); } } @@ -102,24 +107,47 @@ public class NodeHeartbeatRequestPBImpl } @Override - public MasterKey getLastKnownMasterKey() { + public MasterKey getLastKnownContainerTokenMasterKey() { NodeHeartbeatRequestProtoOrBuilder p = viaProto ? proto : builder; - if (this.lastKnownMasterKey != null) { - return this.lastKnownMasterKey; + if (this.lastKnownContainerTokenMasterKey != null) { + return this.lastKnownContainerTokenMasterKey; } - if (!p.hasLastKnownMasterKey()) { + if (!p.hasLastKnownContainerTokenMasterKey()) { return null; } - this.lastKnownMasterKey = convertFromProtoFormat(p.getLastKnownMasterKey()); - return this.lastKnownMasterKey; + this.lastKnownContainerTokenMasterKey = + convertFromProtoFormat(p.getLastKnownContainerTokenMasterKey()); + return this.lastKnownContainerTokenMasterKey; } @Override - public void setLastKnownMasterKey(MasterKey masterKey) { + public void setLastKnownContainerTokenMasterKey(MasterKey masterKey) { maybeInitBuilder(); if (masterKey == null) - builder.clearLastKnownMasterKey(); - this.lastKnownMasterKey = masterKey; + builder.clearLastKnownContainerTokenMasterKey(); + this.lastKnownContainerTokenMasterKey = masterKey; + } + + @Override + public MasterKey getLastKnownNMTokenMasterKey() { + NodeHeartbeatRequestProtoOrBuilder p = viaProto ? proto : builder; + if (this.lastKnownNMTokenMasterKey != null) { + return this.lastKnownNMTokenMasterKey; + } + if (!p.hasLastKnownNmTokenMasterKey()) { + return null; + } + this.lastKnownNMTokenMasterKey = + convertFromProtoFormat(p.getLastKnownNmTokenMasterKey()); + return this.lastKnownNMTokenMasterKey; + } + + @Override + public void setLastKnownNMTokenMasterKey(MasterKey masterKey) { + maybeInitBuilder(); + if (masterKey == null) + builder.clearLastKnownNmTokenMasterKey(); + this.lastKnownNMTokenMasterKey = masterKey; } private NodeStatusPBImpl convertFromProtoFormat(NodeStatusProto p) { Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java?rev=1492915&r1=1492914&r2=1492915&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java Fri Jun 14 00:21:34 2013 @@ -47,7 +47,8 @@ public class NodeHeartbeatResponsePBImpl private List containersToCleanup = null; private List applicationsToCleanup = null; - private MasterKey masterKey = null; + private MasterKey containerTokenMasterKey = null; + private MasterKey nmTokenMasterKey = null; public NodeHeartbeatResponsePBImpl() { builder = NodeHeartbeatResponseProto.newBuilder(); @@ -72,8 +73,13 @@ public class NodeHeartbeatResponsePBImpl if (this.applicationsToCleanup != null) { addApplicationsToCleanupToProto(); } - if (this.masterKey != null) { - builder.setMasterKey(convertToProtoFormat(this.masterKey)); + if (this.containerTokenMasterKey != null) { + builder.setContainerTokenMasterKey( + convertToProtoFormat(this.containerTokenMasterKey)); + } + if (this.nmTokenMasterKey != null) { + builder.setNmTokenMasterKey( + convertToProtoFormat(this.nmTokenMasterKey)); } } @@ -106,24 +112,47 @@ public class NodeHeartbeatResponsePBImpl } @Override - public MasterKey getMasterKey() { + public MasterKey getContainerTokenMasterKey() { NodeHeartbeatResponseProtoOrBuilder p = viaProto ? proto : builder; - if (this.masterKey != null) { - return this.masterKey; + if (this.containerTokenMasterKey != null) { + return this.containerTokenMasterKey; } - if (!p.hasMasterKey()) { + if (!p.hasContainerTokenMasterKey()) { return null; } - this.masterKey = convertFromProtoFormat(p.getMasterKey()); - return this.masterKey; + this.containerTokenMasterKey = + convertFromProtoFormat(p.getContainerTokenMasterKey()); + return this.containerTokenMasterKey; } @Override - public void setMasterKey(MasterKey masterKey) { + public void setContainerTokenMasterKey(MasterKey masterKey) { maybeInitBuilder(); if (masterKey == null) - builder.clearMasterKey(); - this.masterKey = masterKey; + builder.clearContainerTokenMasterKey(); + this.containerTokenMasterKey = masterKey; + } + + @Override + public MasterKey getNMTokenMasterKey() { + NodeHeartbeatResponseProtoOrBuilder p = viaProto ? proto : builder; + if (this.nmTokenMasterKey != null) { + return this.nmTokenMasterKey; + } + if (!p.hasNmTokenMasterKey()) { + return null; + } + this.nmTokenMasterKey = + convertFromProtoFormat(p.getNmTokenMasterKey()); + return this.nmTokenMasterKey; + } + + @Override + public void setNMTokenMasterKey(MasterKey masterKey) { + maybeInitBuilder(); + if (masterKey == null) + builder.clearNmTokenMasterKey(); + this.nmTokenMasterKey = masterKey; } @Override Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerResponsePBImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerResponsePBImpl.java?rev=1492915&r1=1492914&r2=1492915&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerResponsePBImpl.java (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerResponsePBImpl.java Fri Jun 14 00:21:34 2013 @@ -36,7 +36,8 @@ public class RegisterNodeManagerResponse RegisterNodeManagerResponseProto.Builder builder = null; boolean viaProto = false; - private MasterKey masterKey = null; + private MasterKey containerTokenMasterKey = null; + private MasterKey nmTokenMasterKey = null; private boolean rebuild = false; @@ -58,8 +59,13 @@ public class RegisterNodeManagerResponse } private void mergeLocalToBuilder() { - if (this.masterKey != null) { - builder.setMasterKey(convertToProtoFormat(this.masterKey)); + if (this.containerTokenMasterKey != null) { + builder.setContainerTokenMasterKey( + convertToProtoFormat(this.containerTokenMasterKey)); + } + if (this.nmTokenMasterKey != null) { + builder.setNmTokenMasterKey( + convertToProtoFormat(this.nmTokenMasterKey)); } } @@ -80,24 +86,48 @@ public class RegisterNodeManagerResponse } @Override - public MasterKey getMasterKey() { + public MasterKey getContainerTokenMasterKey() { RegisterNodeManagerResponseProtoOrBuilder p = viaProto ? proto : builder; - if (this.masterKey != null) { - return this.masterKey; + if (this.containerTokenMasterKey != null) { + return this.containerTokenMasterKey; } - if (!p.hasMasterKey()) { + if (!p.hasContainerTokenMasterKey()) { return null; } - this.masterKey = convertFromProtoFormat(p.getMasterKey()); - return this.masterKey; + this.containerTokenMasterKey = + convertFromProtoFormat(p.getContainerTokenMasterKey()); + return this.containerTokenMasterKey; } @Override - public void setMasterKey(MasterKey masterKey) { + public void setContainerTokenMasterKey(MasterKey masterKey) { maybeInitBuilder(); if (masterKey == null) - builder.clearMasterKey(); - this.masterKey = masterKey; + builder.clearContainerTokenMasterKey(); + this.containerTokenMasterKey = masterKey; + rebuild = true; + } + + @Override + public MasterKey getNMTokenMasterKey() { + RegisterNodeManagerResponseProtoOrBuilder p = viaProto ? proto : builder; + if (this.nmTokenMasterKey != null) { + return this.nmTokenMasterKey; + } + if (!p.hasNmTokenMasterKey()) { + return null; + } + this.nmTokenMasterKey = + convertFromProtoFormat(p.getNmTokenMasterKey()); + return this.nmTokenMasterKey; + } + + @Override + public void setNMTokenMasterKey(MasterKey masterKey) { + maybeInitBuilder(); + if (masterKey == null) + builder.clearNmTokenMasterKey(); + this.nmTokenMasterKey = masterKey; rebuild = true; } Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java?rev=1492915&r1=1492914&r2=1492915&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java Fri Jun 14 00:21:34 2013 @@ -60,37 +60,6 @@ public class BaseContainerTokenSecretMan */ protected MasterKeyData currentMasterKey; - protected final class MasterKeyData { - - private final MasterKey masterKeyRecord; - // Underlying secret-key also stored to avoid repetitive encoding and - // decoding the masterKeyRecord bytes. - private final SecretKey generatedSecretKey; - - private MasterKeyData() { - this.masterKeyRecord = Records.newRecord(MasterKey.class); - this.masterKeyRecord.setKeyId(serialNo++); - this.generatedSecretKey = generateSecret(); - this.masterKeyRecord.setBytes(ByteBuffer.wrap(generatedSecretKey - .getEncoded())); - } - - public MasterKeyData(MasterKey masterKeyRecord) { - this.masterKeyRecord = masterKeyRecord; - this.generatedSecretKey = - SecretManager.createSecretKey(this.masterKeyRecord.getBytes().array() - .clone()); - } - - public MasterKey getMasterKey() { - return this.masterKeyRecord; - } - - private SecretKey getSecretKey() { - return this.generatedSecretKey; - } - } - protected final long containerTokenExpiryInterval; public BaseContainerTokenSecretManager(Configuration conf) { @@ -103,7 +72,7 @@ public class BaseContainerTokenSecretMan protected MasterKeyData createNewMasterKey() { this.writeLock.lock(); try { - return new MasterKeyData(); + return new MasterKeyData(serialNo++, generateSecret()); } finally { this.writeLock.unlock(); } Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java?rev=1492915&r1=1492914&r2=1492915&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java Fri Jun 14 00:21:34 2013 @@ -48,7 +48,6 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; -import org.apache.hadoop.yarn.api.records.NodeHealthStatus; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.NodeState; Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java?rev=1492915&r1=1492914&r2=1492915&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java Fri Jun 14 00:21:34 2013 @@ -42,12 +42,14 @@ public class YarnServerBuilderUtils { public static NodeHeartbeatResponse newNodeHeartbeatResponse(int responseId, NodeAction action, List containersToCleanUp, List applicationsToCleanUp, - MasterKey masterKey, long nextHeartbeatInterval) { + MasterKey containerTokenMasterKey, MasterKey nmTokenMasterKey, + long nextHeartbeatInterval) { NodeHeartbeatResponse response = recordFactory .newRecordInstance(NodeHeartbeatResponse.class); response.setResponseId(responseId); response.setNodeAction(action); - response.setMasterKey(masterKey); + response.setContainerTokenMasterKey(containerTokenMasterKey); + response.setNMTokenMasterKey(nmTokenMasterKey); response.setNextHeartBeatInterval(nextHeartbeatInterval); if(containersToCleanUp != null) { response.addAllContainersToCleanup(containersToCleanUp); Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto?rev=1492915&r1=1492914&r2=1492915&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto Fri Jun 14 00:21:34 2013 @@ -31,24 +31,27 @@ message RegisterNodeManagerRequestProto } message RegisterNodeManagerResponseProto { - optional MasterKeyProto master_key = 1; - optional NodeActionProto nodeAction = 2; - optional int64 rm_identifier = 3; - optional string diagnostics_message = 4; + optional MasterKeyProto container_token_master_key = 1; + optional MasterKeyProto nm_token_master_key = 2; + optional NodeActionProto nodeAction = 3; + optional int64 rm_identifier = 4; + optional string diagnostics_message = 5; } message NodeHeartbeatRequestProto { optional NodeStatusProto node_status = 1; - optional MasterKeyProto last_known_master_key = 2; + optional MasterKeyProto last_known_container_token_master_key = 2; + optional MasterKeyProto last_known_nm_token_master_key = 3; } message NodeHeartbeatResponseProto { optional int32 response_id = 1; - optional MasterKeyProto master_key = 2; - optional NodeActionProto nodeAction = 3; - repeated ContainerIdProto containers_to_cleanup = 4; - repeated ApplicationIdProto applications_to_cleanup = 5; - optional int64 nextHeartBeatInterval = 6; - optional string diagnostics_message = 7; + optional MasterKeyProto container_token_master_key = 2; + optional MasterKeyProto nm_token_master_key = 3; + optional NodeActionProto nodeAction = 4; + repeated ContainerIdProto containers_to_cleanup = 5; + repeated ApplicationIdProto applications_to_cleanup = 6; + optional int64 nextHeartBeatInterval = 7; + optional string diagnostics_message = 8; } Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/api/protocolrecords/TestRegisterNodeManagerResponse.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/api/protocolrecords/TestRegisterNodeManagerResponse.java?rev=1492915&r1=1492914&r2=1492915&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/api/protocolrecords/TestRegisterNodeManagerResponse.java (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/api/protocolrecords/TestRegisterNodeManagerResponse.java Fri Jun 14 00:21:34 2013 @@ -41,24 +41,50 @@ public class TestRegisterNodeManagerResp public void testRoundTrip() throws Exception { RegisterNodeManagerResponse resp = recordFactory .newRecordInstance(RegisterNodeManagerResponse.class); - MasterKey mk = recordFactory.newRecordInstance(MasterKey.class); - mk.setKeyId(54321); + byte b [] = {0,1,2,3,4,5}; - mk.setBytes(ByteBuffer.wrap(b)); - resp.setMasterKey(mk); - resp.setNodeAction(NodeAction.NORMAL); + MasterKey containerTokenMK = + recordFactory.newRecordInstance(MasterKey.class); + containerTokenMK.setKeyId(54321); + containerTokenMK.setBytes(ByteBuffer.wrap(b)); + resp.setContainerTokenMasterKey(containerTokenMK); + + MasterKey nmTokenMK = + recordFactory.newRecordInstance(MasterKey.class); + nmTokenMK.setKeyId(12345); + nmTokenMK.setBytes(ByteBuffer.wrap(b)); + resp.setNMTokenMasterKey(nmTokenMK); + + resp.setNodeAction(NodeAction.NORMAL); + assertEquals(NodeAction.NORMAL, resp.getNodeAction()); - assertNotNull(resp.getMasterKey()); - assertEquals(54321, resp.getMasterKey().getKeyId()); - assertArrayEquals(b, resp.getMasterKey().getBytes().array()); + + // Verifying containerTokenMasterKey + assertNotNull(resp.getContainerTokenMasterKey()); + assertEquals(54321, resp.getContainerTokenMasterKey().getKeyId()); + assertArrayEquals(b, resp.getContainerTokenMasterKey().getBytes().array()); RegisterNodeManagerResponse respCopy = serDe(resp); assertEquals(NodeAction.NORMAL, respCopy.getNodeAction()); - assertNotNull(respCopy.getMasterKey()); - assertEquals(54321, respCopy.getMasterKey().getKeyId()); - assertArrayEquals(b, respCopy.getMasterKey().getBytes().array()); + assertNotNull(respCopy.getContainerTokenMasterKey()); + assertEquals(54321, respCopy.getContainerTokenMasterKey().getKeyId()); + assertArrayEquals(b, respCopy.getContainerTokenMasterKey().getBytes() + .array()); + + // Verifying nmTokenMasterKey + assertNotNull(resp.getNMTokenMasterKey()); + assertEquals(12345, resp.getNMTokenMasterKey().getKeyId()); + assertArrayEquals(b, resp.getNMTokenMasterKey().getBytes().array()); + + respCopy = serDe(resp); + + assertEquals(NodeAction.NORMAL, respCopy.getNodeAction()); + assertNotNull(respCopy.getNMTokenMasterKey()); + assertEquals(12345, respCopy.getNMTokenMasterKey().getKeyId()); + assertArrayEquals(b, respCopy.getNMTokenMasterKey().getBytes().array()); + } public static RegisterNodeManagerResponse serDe(RegisterNodeManagerResponse orig) throws Exception { Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java?rev=1492915&r1=1492914&r2=1492915&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java Fri Jun 14 00:21:34 2013 @@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager; +import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM; /** * Context interface for sharing information across components in the @@ -54,6 +55,8 @@ public interface Context { ConcurrentMap getContainers(); NMContainerTokenSecretManager getContainerTokenSecretManager(); + + NMTokenSecretManagerInNM getNMTokenSecretManager(); NodeHealthStatus getNodeHealthStatus(); Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java?rev=1492915&r1=1492914&r2=1492915&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java Fri Jun 14 00:21:34 2013 @@ -52,6 +52,7 @@ import org.apache.hadoop.yarn.server.nod import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics; import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager; +import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM; import org.apache.hadoop.yarn.server.nodemanager.webapp.WebServer; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.service.CompositeService; @@ -118,8 +119,10 @@ public class NodeManager extends Composi return new DeletionService(exec); } - protected NMContext createNMContext(NMContainerTokenSecretManager containerTokenSecretManager) { - return new NMContext(containerTokenSecretManager); + protected NMContext createNMContext( + NMContainerTokenSecretManager containerTokenSecretManager, + NMTokenSecretManagerInNM nmTokenSecretManager) { + return new NMContext(containerTokenSecretManager, nmTokenSecretManager); } protected void doSecureLogin() throws IOException { @@ -135,7 +138,11 @@ public class NodeManager extends Composi NMContainerTokenSecretManager containerTokenSecretManager = new NMContainerTokenSecretManager(conf); - this.context = createNMContext(containerTokenSecretManager); + NMTokenSecretManagerInNM nmTokenSecretManager = + new NMTokenSecretManagerInNM(); + + this.context = + createNMContext(containerTokenSecretManager, nmTokenSecretManager); this.aclsManager = new ApplicationACLsManager(conf); @@ -299,13 +306,16 @@ public class NodeManager extends Composi new ConcurrentSkipListMap(); private final NMContainerTokenSecretManager containerTokenSecretManager; + private final NMTokenSecretManagerInNM nmTokenSecretManager; private ContainerManager containerManager; private WebServer webServer; private final NodeHealthStatus nodeHealthStatus = RecordFactoryProvider .getRecordFactory(null).newRecordInstance(NodeHealthStatus.class); - public NMContext(NMContainerTokenSecretManager containerTokenSecretManager) { + public NMContext(NMContainerTokenSecretManager containerTokenSecretManager, + NMTokenSecretManagerInNM nmTokenSecretManager) { this.containerTokenSecretManager = containerTokenSecretManager; + this.nmTokenSecretManager = nmTokenSecretManager; this.nodeHealthStatus.setIsNodeHealthy(true); this.nodeHealthStatus.setHealthReport("Healthy"); this.nodeHealthStatus.setLastHealthReportTime(System.currentTimeMillis()); @@ -338,6 +348,12 @@ public class NodeManager extends Composi public NMContainerTokenSecretManager getContainerTokenSecretManager() { return this.containerTokenSecretManager; } + + @Override + public NMTokenSecretManagerInNM getNMTokenSecretManager() { + return this.nmTokenSecretManager; + } + @Override public NodeHealthStatus getNodeHealthStatus() { return this.nodeHealthStatus; Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java?rev=1492915&r1=1492914&r2=1492915&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java Fri Jun 14 00:21:34 2013 @@ -297,7 +297,7 @@ public class NodeStatusUpdaterImpl exten + message); } - MasterKey masterKey = regNMResponse.getMasterKey(); + MasterKey masterKey = regNMResponse.getContainerTokenMasterKey(); // do this now so that its set before we start heartbeating to RM // It is expected that status updater is started by this point and // RM gives the shared secret in registration during @@ -305,6 +305,11 @@ public class NodeStatusUpdaterImpl exten if (masterKey != null) { this.context.getContainerTokenSecretManager().setMasterKey(masterKey); } + + masterKey = regNMResponse.getNMTokenMasterKey(); + if (masterKey != null) { + this.context.getNMTokenSecretManager().setMasterKey(masterKey); + } LOG.info("Registered with ResourceManager as " + this.nodeId + " with total resource of " + this.totalResource); @@ -434,8 +439,12 @@ public class NodeStatusUpdaterImpl exten NodeHeartbeatRequest request = recordFactory .newRecordInstance(NodeHeartbeatRequest.class); request.setNodeStatus(nodeStatus); - request.setLastKnownMasterKey(NodeStatusUpdaterImpl.this.context - .getContainerTokenSecretManager().getCurrentKey()); + request + .setLastKnownContainerTokenMasterKey(NodeStatusUpdaterImpl.this.context + .getContainerTokenSecretManager().getCurrentKey()); + request + .setLastKnownNMTokenMasterKey(NodeStatusUpdaterImpl.this.context + .getNMTokenSecretManager().getCurrentKey()); while (!isStopped) { try { rmRetryCount++; @@ -463,13 +472,7 @@ public class NodeStatusUpdaterImpl exten } //get next heartbeat interval from response nextHeartBeatInterval = response.getNextHeartBeatInterval(); - // See if the master-key has rolled over - MasterKey updatedMasterKey = response.getMasterKey(); - if (updatedMasterKey != null) { - // Will be non-null only on roll-over on RM side - context.getContainerTokenSecretManager().setMasterKey( - updatedMasterKey); - } + updateMasterKeys(response); if (response.getNodeAction() == NodeAction.SHUTDOWN) { LOG @@ -533,6 +536,20 @@ public class NodeStatusUpdaterImpl exten } } } + + private void updateMasterKeys(NodeHeartbeatResponse response) { + // See if the master-key has rolled over + MasterKey updatedMasterKey = response.getContainerTokenMasterKey(); + if (updatedMasterKey != null) { + // Will be non-null only on roll-over on RM side + context.getContainerTokenSecretManager().setMasterKey(updatedMasterKey); + } + + updatedMasterKey = response.getNMTokenMasterKey(); + if (updatedMasterKey != null) { + context.getNMTokenSecretManager().setMasterKey(updatedMasterKey); + } + } }; statusUpdater = new Thread(statusUpdaterRunnable, "Node Status Updater"); Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMContainerTokenSecretManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMContainerTokenSecretManager.java?rev=1492915&r1=1492914&r2=1492915&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMContainerTokenSecretManager.java (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMContainerTokenSecretManager.java Fri Jun 14 00:21:34 2013 @@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.server.api.records.MasterKey; import org.apache.hadoop.yarn.server.security.BaseContainerTokenSecretManager; +import org.apache.hadoop.yarn.server.security.MasterKeyData; import com.google.common.annotations.VisibleForTesting; @@ -69,13 +70,17 @@ public class NMContainerTokenSecretManag LOG.info("Rolling master-key for container-tokens, got key with id " + masterKeyRecord.getKeyId()); if (super.currentMasterKey == null) { - super.currentMasterKey = new MasterKeyData(masterKeyRecord); + super.currentMasterKey = + new MasterKeyData(masterKeyRecord, createSecretKey(masterKeyRecord + .getBytes().array())); } else { if (super.currentMasterKey.getMasterKey().getKeyId() != masterKeyRecord .getKeyId()) { // Update keys only if the key has changed. this.previousMasterKey = super.currentMasterKey; - super.currentMasterKey = new MasterKeyData(masterKeyRecord); + super.currentMasterKey = + new MasterKeyData(masterKeyRecord, createSecretKey(masterKeyRecord + .getBytes().array())); } } } Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/LocalRMInterface.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/LocalRMInterface.java?rev=1492915&r1=1492914&r2=1492915&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/LocalRMInterface.java (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/LocalRMInterface.java Fri Jun 14 00:21:34 2013 @@ -46,7 +46,8 @@ public class LocalRMInterface implements masterKey.setKeyId(123); masterKey.setBytes(ByteBuffer.wrap(new byte[] { new Integer(123) .byteValue() })); - response.setMasterKey(masterKey); + response.setContainerTokenMasterKey(masterKey); + response.setNMTokenMasterKey(masterKey); return response; } Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/MockNodeStatusUpdater.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/MockNodeStatusUpdater.java?rev=1492915&r1=1492914&r2=1492915&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/MockNodeStatusUpdater.java (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/MockNodeStatusUpdater.java Fri Jun 14 00:21:34 2013 @@ -75,7 +75,8 @@ public class MockNodeStatusUpdater exten masterKey.setKeyId(123); masterKey.setBytes(ByteBuffer.wrap(new byte[] { new Integer(123) .byteValue() })); - response.setMasterKey(masterKey); + response.setContainerTokenMasterKey(masterKey); + response.setNMTokenMasterKey(masterKey); return response; } @@ -88,7 +89,7 @@ public class MockNodeStatusUpdater exten NodeHeartbeatResponse nhResponse = YarnServerBuilderUtils .newNodeHeartbeatResponse(heartBeatID, null, null, - null, null, 1000L); + null, null, null, 1000L); return nhResponse; } } Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java?rev=1492915&r1=1492914&r2=1492915&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java Fri Jun 14 00:21:34 2013 @@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.server.nod import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest; import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics; import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager; +import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.junit.Test; @@ -76,7 +77,8 @@ public class TestEventFlow { YarnConfiguration conf = new YarnConfiguration(); - Context context = new NMContext(new NMContainerTokenSecretManager(conf)) { + Context context = new NMContext(new NMContainerTokenSecretManager(conf), + new NMTokenSecretManagerInNM()) { @Override public int getHttpPort() { return 1234; Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java?rev=1492915&r1=1492914&r2=1492915&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java Fri Jun 14 00:21:34 2013 @@ -76,6 +76,7 @@ import org.apache.hadoop.yarn.server.nod import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl; import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics; import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager; +import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.server.utils.YarnServerBuilderUtils; @@ -147,7 +148,8 @@ public class TestNodeStatusUpdater { RegisterNodeManagerResponse response = recordFactory .newRecordInstance(RegisterNodeManagerResponse.class); - response.setMasterKey(createMasterKey()); + response.setContainerTokenMasterKey(createMasterKey()); + response.setNMTokenMasterKey(createMasterKey()); return response; } @@ -251,7 +253,8 @@ public class TestNodeStatusUpdater { } NodeHeartbeatResponse nhResponse = YarnServerBuilderUtils. - newNodeHeartbeatResponse(heartBeatID, null, null, null, null, 1000L); + newNodeHeartbeatResponse(heartBeatID, null, null, null, null, null, + 1000L); return nhResponse; } } @@ -447,7 +450,8 @@ public class TestNodeStatusUpdater { RegisterNodeManagerResponse response = recordFactory .newRecordInstance(RegisterNodeManagerResponse.class); response.setNodeAction(registerNodeAction ); - response.setMasterKey(createMasterKey()); + response.setContainerTokenMasterKey(createMasterKey()); + response.setNMTokenMasterKey(createMasterKey()); response.setDiagnosticsMessage(shutDownMessage); return response; } @@ -459,7 +463,7 @@ public class TestNodeStatusUpdater { NodeHeartbeatResponse nhResponse = YarnServerBuilderUtils. newNodeHeartbeatResponse(heartBeatID, heartBeatNodeAction, null, - null, null, 1000L); + null, null, null, 1000L); nhResponse.setDiagnosticsMessage(shutDownMessage); return nhResponse; } @@ -485,7 +489,8 @@ public class TestNodeStatusUpdater { RegisterNodeManagerResponse response = recordFactory.newRecordInstance(RegisterNodeManagerResponse.class); response.setNodeAction(registerNodeAction); - response.setMasterKey(createMasterKey()); + response.setContainerTokenMasterKey(createMasterKey()); + response.setNMTokenMasterKey(createMasterKey()); return response; } @@ -497,7 +502,7 @@ public class TestNodeStatusUpdater { nodeStatus.setResponseId(heartBeatID++); NodeHeartbeatResponse nhResponse = YarnServerBuilderUtils. newNodeHeartbeatResponse(heartBeatID, heartBeatNodeAction, null, - null, null, 1000L); + null, null, null, 1000L); if (nodeStatus.getKeepAliveApplications() != null && nodeStatus.getKeepAliveApplications().size() > 0) { @@ -536,7 +541,8 @@ public class TestNodeStatusUpdater { RegisterNodeManagerResponse response = recordFactory .newRecordInstance(RegisterNodeManagerResponse.class); response.setNodeAction(registerNodeAction); - response.setMasterKey(createMasterKey()); + response.setContainerTokenMasterKey(createMasterKey()); + response.setNMTokenMasterKey(createMasterKey()); return response; } @@ -616,7 +622,7 @@ public class TestNodeStatusUpdater { YarnServerBuilderUtils.newNodeHeartbeatResponse(heartBeatID, heartBeatNodeAction, null, null, null, - 1000L); + null, 1000L); return nhResponse; } } @@ -631,8 +637,8 @@ public class TestNodeStatusUpdater { RegisterNodeManagerResponse response = recordFactory .newRecordInstance(RegisterNodeManagerResponse.class); response.setNodeAction(registerNodeAction ); - response.setMasterKey(createMasterKey()); - + response.setContainerTokenMasterKey(createMasterKey()); + response.setNMTokenMasterKey(createMasterKey()); return response; } @@ -1004,10 +1010,11 @@ public class TestNodeStatusUpdater { @Override protected NMContext createNMContext( - NMContainerTokenSecretManager containerTokenSecretManager) { - return new MyNMContext(containerTokenSecretManager); + NMContainerTokenSecretManager containerTokenSecretManager, + NMTokenSecretManagerInNM nmTokenSecretManager) { + return new MyNMContext(containerTokenSecretManager, + nmTokenSecretManager); } - }; YarnConfiguration conf = createNMConfig(); @@ -1052,9 +1059,10 @@ public class TestNodeStatusUpdater { ConcurrentMap containers = new ConcurrentSkipListMap(); - public MyNMContext(NMContainerTokenSecretManager - containerTokenSecretManager) { - super(containerTokenSecretManager); + public MyNMContext( + NMContainerTokenSecretManager containerTokenSecretManager, + NMTokenSecretManagerInNM nmTokenSecretManager) { + super(containerTokenSecretManager, nmTokenSecretManager); } @Override Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java?rev=1492915&r1=1492914&r2=1492915&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java Fri Jun 14 00:21:34 2013 @@ -59,6 +59,7 @@ import org.apache.hadoop.yarn.server.nod import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState; import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics; import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager; +import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.service.Service.STATE; import org.junit.After; @@ -97,7 +98,7 @@ public abstract class BaseContainerManag protected static final int HTTP_PORT = 5412; protected Configuration conf = new YarnConfiguration(); protected Context context = new NMContext(new NMContainerTokenSecretManager( - conf)) { + conf), new NMTokenSecretManagerInNM()) { public int getHttpPort() { return HTTP_PORT; }; Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java?rev=1492915&r1=1492914&r2=1492915&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java Fri Jun 14 00:21:34 2013 @@ -76,7 +76,7 @@ public class TestNMWebServer { } private int startNMWebAppServer(String webAddr) { - Context nmContext = new NodeManager.NMContext(null); + Context nmContext = new NodeManager.NMContext(null, null); ResourceView resourceView = new ResourceView() { @Override public long getVmemAllocatedForContainers() { @@ -134,7 +134,7 @@ public class TestNMWebServer { @Test public void testNMWebApp() throws IOException { - Context nmContext = new NodeManager.NMContext(null); + Context nmContext = new NodeManager.NMContext(null, null); ResourceView resourceView = new ResourceView() { @Override public long getVmemAllocatedForContainers() { Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java?rev=1492915&r1=1492914&r2=1492915&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java Fri Jun 14 00:21:34 2013 @@ -86,7 +86,7 @@ public class TestNMWebServices extends J private Injector injector = Guice.createInjector(new ServletModule() { @Override protected void configureServlets() { - nmContext = new NodeManager.NMContext(null); + nmContext = new NodeManager.NMContext(null, null); NodeId nodeId = NodeId.newInstance("testhost.foo.com", 8042); ((NodeManager.NMContext)nmContext).setNodeId(nodeId); resourceView = new ResourceView() { Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java?rev=1492915&r1=1492914&r2=1492915&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java Fri Jun 14 00:21:34 2013 @@ -93,7 +93,7 @@ public class TestNMWebServicesApps exten private Injector injector = Guice.createInjector(new ServletModule() { @Override protected void configureServlets() { - nmContext = new NodeManager.NMContext(null); + nmContext = new NodeManager.NMContext(null, null); NodeId nodeId = NodeId.newInstance("testhost.foo.com", 9999); ((NodeManager.NMContext)nmContext).setNodeId(nodeId); resourceView = new ResourceView() { Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java?rev=1492915&r1=1492914&r2=1492915&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java Fri Jun 14 00:21:34 2013 @@ -93,7 +93,7 @@ public class TestNMWebServicesContainers private Injector injector = Guice.createInjector(new ServletModule() { @Override protected void configureServlets() { - nmContext = new NodeManager.NMContext(null) { + nmContext = new NodeManager.NMContext(null, null) { public NodeId getNodeId() { return NodeId.newInstance("testhost.foo.com", 8042); }; Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java?rev=1492915&r1=1492914&r2=1492915&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java Fri Jun 14 00:21:34 2013 @@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; +import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; /** * Context of the ResourceManager. @@ -59,6 +60,8 @@ public interface RMContext { ApplicationTokenSecretManager getApplicationTokenSecretManager(); RMContainerTokenSecretManager getContainerTokenSecretManager(); + + NMTokenSecretManagerInRM getNMTokenSecretManager(); ClientToAMTokenSecretManagerInRM getClientToAMTokenSecretManager(); } \ No newline at end of file Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java?rev=1492915&r1=1492914&r2=1492915&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java Fri Jun 14 00:21:34 2013 @@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; +import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; import com.google.common.annotations.VisibleForTesting; @@ -58,6 +59,7 @@ public class RMContextImpl implements RM private final DelegationTokenRenewer tokenRenewer; private final ApplicationTokenSecretManager appTokenSecretManager; private final RMContainerTokenSecretManager containerTokenSecretManager; + private final NMTokenSecretManagerInRM nmTokenSecretManager; private final ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager; public RMContextImpl(Dispatcher rmDispatcher, @@ -68,6 +70,7 @@ public class RMContextImpl implements RM DelegationTokenRenewer tokenRenewer, ApplicationTokenSecretManager appTokenSecretManager, RMContainerTokenSecretManager containerTokenSecretManager, + NMTokenSecretManagerInRM nmTokenSecretManager, ClientToAMTokenSecretManagerInRM clientTokenSecretManager) { this.rmDispatcher = rmDispatcher; this.stateStore = store; @@ -77,6 +80,7 @@ public class RMContextImpl implements RM this.tokenRenewer = tokenRenewer; this.appTokenSecretManager = appTokenSecretManager; this.containerTokenSecretManager = containerTokenSecretManager; + this.nmTokenSecretManager = nmTokenSecretManager; this.clientToAMTokenSecretManager = clientTokenSecretManager; } @@ -89,10 +93,12 @@ public class RMContextImpl implements RM DelegationTokenRenewer tokenRenewer, ApplicationTokenSecretManager appTokenSecretManager, RMContainerTokenSecretManager containerTokenSecretManager, + NMTokenSecretManagerInRM nmTokenSecretManager, ClientToAMTokenSecretManagerInRM clientTokenSecretManager) { this(rmDispatcher, null, containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor, tokenRenewer, appTokenSecretManager, - containerTokenSecretManager, clientTokenSecretManager); + containerTokenSecretManager, nmTokenSecretManager, + clientTokenSecretManager); RMStateStore nullStore = new NullRMStateStore(); nullStore.setDispatcher(rmDispatcher); try { @@ -157,7 +163,12 @@ public class RMContextImpl implements RM public RMContainerTokenSecretManager getContainerTokenSecretManager() { return this.containerTokenSecretManager; } - + + @Override + public NMTokenSecretManagerInRM getNMTokenSecretManager() { + return this.nmTokenSecretManager; + } + @Override public ClientToAMTokenSecretManagerInRM getClientToAMTokenSecretManager() { return this.clientToAMTokenSecretManager; Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java?rev=1492915&r1=1492914&r2=1492915&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java Fri Jun 14 00:21:34 2013 @@ -69,6 +69,7 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager; +import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebApp; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.webproxy.AppReportFetcher; @@ -104,6 +105,7 @@ public class ResourceManager extends Com new ClientToAMTokenSecretManagerInRM(); protected RMContainerTokenSecretManager containerTokenSecretManager; + protected NMTokenSecretManagerInRM nmTokenSecretManager; protected ApplicationTokenSecretManager appTokenSecretManager; @@ -164,6 +166,7 @@ public class ResourceManager extends Com addService(tokenRenewer); this.containerTokenSecretManager = createContainerTokenSecretManager(conf); + this.nmTokenSecretManager = createNMTokenSecretManager(conf); boolean isRecoveryEnabled = conf.getBoolean( YarnConfiguration.RECOVERY_ENABLED, @@ -191,7 +194,8 @@ public class ResourceManager extends Com new RMContextImpl(this.rmDispatcher, rmStore, this.containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor, tokenRenewer, this.appTokenSecretManager, - this.containerTokenSecretManager, this.clientToAMSecretManager); + this.containerTokenSecretManager, this.nmTokenSecretManager, + this.clientToAMSecretManager); // Register event handler for NodesListManager this.nodesListManager = new NodesListManager(this.rmContext); @@ -271,6 +275,11 @@ public class ResourceManager extends Com return new RMContainerTokenSecretManager(conf); } + protected NMTokenSecretManagerInRM createNMTokenSecretManager( + Configuration conf) { + return new NMTokenSecretManagerInRM(conf); + } + protected EventHandler createSchedulerEventDispatcher() { return new SchedulerEventDispatcher(this.scheduler); } @@ -586,6 +595,7 @@ public class ResourceManager extends Com this.appTokenSecretManager.start(); this.containerTokenSecretManager.start(); + this.nmTokenSecretManager.start(); if(recoveryEnabled) { try { @@ -649,6 +659,9 @@ public class ResourceManager extends Com if (containerTokenSecretManager != null) { this.containerTokenSecretManager.stop(); } + if(nmTokenSecretManager != null) { + nmTokenSecretManager.stop(); + } /*synchronized(shutdown) { shutdown.set(true); @@ -671,7 +684,8 @@ public class ResourceManager extends Com protected ResourceTrackerService createResourceTrackerService() { return new ResourceTrackerService(this.rmContext, this.nodesListManager, - this.nmLivelinessMonitor, this.containerTokenSecretManager); + this.nmLivelinessMonitor, this.containerTokenSecretManager, + this.nmTokenSecretManager); } protected RMDelegationTokenSecretManager @@ -748,6 +762,11 @@ public class ResourceManager extends Com } @Private + public NMTokenSecretManagerInRM getRMNMTokenSecretManager() { + return this.nmTokenSecretManager; + } + + @Private public ApplicationTokenSecretManager getApplicationTokenSecretManager(){ return this.appTokenSecretManager; }