Return-Path: X-Original-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id E15459057 for ; Wed, 18 Apr 2012 23:46:24 +0000 (UTC) Received: (qmail 50600 invoked by uid 500); 18 Apr 2012 23:46:24 -0000 Delivered-To: apmail-hadoop-mapreduce-commits-archive@hadoop.apache.org Received: (qmail 50542 invoked by uid 500); 18 Apr 2012 23:46:24 -0000 Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: mapreduce-dev@hadoop.apache.org Delivered-To: mailing list mapreduce-commits@hadoop.apache.org Received: (qmail 50533 invoked by uid 99); 18 Apr 2012 23:46:24 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 18 Apr 2012 23:46:24 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 18 Apr 2012 23:36:28 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 08D782388C1C; Wed, 18 Apr 2012 23:35:48 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1327724 [3/4] - in /hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project: ./ conf/ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/ hadoop-mapreduce-client/hadoop-mapreduce-client-a... Date: Wed, 18 Apr 2012 23:35:42 -0000 To: mapreduce-commits@hadoop.apache.org From: todd@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120418233548.08D782388C1C@eris.apache.org> Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/ipc/TestSocketFactory.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/ipc/TestSocketFactory.java?rev=1327724&r1=1327723&r2=1327724&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/ipc/TestSocketFactory.java (original) +++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/ipc/TestSocketFactory.java Wed Apr 18 23:35:30 2012 @@ -22,124 +22,149 @@ import java.net.InetSocketAddress; import java.net.Socket; import java.net.SocketAddress; -import junit.framework.TestCase; - import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.mapred.JobClient; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.JobStatus; -import org.apache.hadoop.mapred.MiniMRCluster; import org.apache.hadoop.mapreduce.MRConfig; +import org.apache.hadoop.mapreduce.v2.MiniMRYarnCluster; import org.apache.hadoop.net.StandardSocketFactory; -import org.junit.Ignore; +import org.junit.Assert; +import org.junit.Test; /** * This class checks that RPCs can use specialized socket factories. */ -@Ignore -public class TestSocketFactory extends TestCase { +public class TestSocketFactory { /** - * Check that we can reach a NameNode or a JobTracker using a specific + * Check that we can reach a NameNode or Resource Manager using a specific * socket factory */ + @Test public void testSocketFactory() throws IOException { // Create a standard mini-cluster Configuration sconf = new Configuration(); - MiniDFSCluster cluster = new MiniDFSCluster(sconf, 1, true, null); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(sconf).numDataNodes(1) + .build(); final int nameNodePort = cluster.getNameNodePort(); // Get a reference to its DFS directly FileSystem fs = cluster.getFileSystem(); - assertTrue(fs instanceof DistributedFileSystem); + Assert.assertTrue(fs instanceof DistributedFileSystem); DistributedFileSystem directDfs = (DistributedFileSystem) fs; - // Get another reference via network using a specific socket factory - Configuration cconf = new Configuration(); - FileSystem.setDefaultUri(cconf, String.format("hdfs://localhost:%s/", - nameNodePort + 10)); - cconf.set("hadoop.rpc.socket.factory.class.default", - "org.apache.hadoop.ipc.DummySocketFactory"); - cconf.set("hadoop.rpc.socket.factory.class.ClientProtocol", - "org.apache.hadoop.ipc.DummySocketFactory"); - cconf.set("hadoop.rpc.socket.factory.class.JobSubmissionProtocol", - "org.apache.hadoop.ipc.DummySocketFactory"); + Configuration cconf = getCustomSocketConfigs(nameNodePort); fs = FileSystem.get(cconf); - assertTrue(fs instanceof DistributedFileSystem); + Assert.assertTrue(fs instanceof DistributedFileSystem); DistributedFileSystem dfs = (DistributedFileSystem) fs; JobClient client = null; - MiniMRCluster mr = null; + MiniMRYarnCluster miniMRYarnCluster = null; try { // This will test RPC to the NameNode only. // could we test Client-DataNode connections? Path filePath = new Path("/dir"); - assertFalse(directDfs.exists(filePath)); - assertFalse(dfs.exists(filePath)); + Assert.assertFalse(directDfs.exists(filePath)); + Assert.assertFalse(dfs.exists(filePath)); directDfs.mkdirs(filePath); - assertTrue(directDfs.exists(filePath)); - assertTrue(dfs.exists(filePath)); + Assert.assertTrue(directDfs.exists(filePath)); + Assert.assertTrue(dfs.exists(filePath)); - // This will test TPC to a JobTracker + // This will test RPC to a Resource Manager fs = FileSystem.get(sconf); - mr = new MiniMRCluster(1, fs.getUri().toString(), 1); - final int jobTrackerPort = mr.getJobTrackerPort(); - + JobConf jobConf = new JobConf(); + FileSystem.setDefaultUri(jobConf, fs.getUri().toString()); + miniMRYarnCluster = initAndStartMiniMRYarnCluster(jobConf); JobConf jconf = new JobConf(cconf); - jconf.set("mapred.job.tracker", String.format("localhost:%d", - jobTrackerPort + 10)); - jconf.set(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME); + jconf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME); + String rmAddress = jconf.get("yarn.resourcemanager.address"); + String[] split = rmAddress.split(":"); + jconf.set("yarn.resourcemanager.address", split[0] + ':' + + (Integer.parseInt(split[1]) + 10)); client = new JobClient(jconf); JobStatus[] jobs = client.jobsToComplete(); - assertTrue(jobs.length == 0); + Assert.assertTrue(jobs.length == 0); } finally { - try { - if (client != null) - client.close(); - } catch (Exception ignored) { - // nothing we can do - ignored.printStackTrace(); - } - try { - if (dfs != null) - dfs.close(); - - } catch (Exception ignored) { - // nothing we can do - ignored.printStackTrace(); - } - try { - if (directDfs != null) - directDfs.close(); - - } catch (Exception ignored) { - // nothing we can do - ignored.printStackTrace(); - } - try { - if (cluster != null) - cluster.shutdown(); - - } catch (Exception ignored) { - // nothing we can do - ignored.printStackTrace(); - } - if (mr != null) { - try { - mr.shutdown(); - } catch (Exception ignored) { - ignored.printStackTrace(); - } - } + closeClient(client); + closeDfs(dfs); + closeDfs(directDfs); + stopMiniMRYarnCluster(miniMRYarnCluster); + shutdownDFSCluster(cluster); + } + } + + private MiniMRYarnCluster initAndStartMiniMRYarnCluster(JobConf jobConf) { + MiniMRYarnCluster miniMRYarnCluster; + miniMRYarnCluster = new MiniMRYarnCluster(this.getClass().getName(), 1); + miniMRYarnCluster.init(jobConf); + miniMRYarnCluster.start(); + return miniMRYarnCluster; + } + + private Configuration getCustomSocketConfigs(final int nameNodePort) { + // Get another reference via network using a specific socket factory + Configuration cconf = new Configuration(); + FileSystem.setDefaultUri(cconf, String.format("hdfs://localhost:%s/", + nameNodePort + 10)); + cconf.set("hadoop.rpc.socket.factory.class.default", + "org.apache.hadoop.ipc.DummySocketFactory"); + cconf.set("hadoop.rpc.socket.factory.class.ClientProtocol", + "org.apache.hadoop.ipc.DummySocketFactory"); + cconf.set("hadoop.rpc.socket.factory.class.JobSubmissionProtocol", + "org.apache.hadoop.ipc.DummySocketFactory"); + return cconf; + } + + private void shutdownDFSCluster(MiniDFSCluster cluster) { + try { + if (cluster != null) + cluster.shutdown(); + + } catch (Exception ignored) { + // nothing we can do + ignored.printStackTrace(); + } + } + + private void stopMiniMRYarnCluster(MiniMRYarnCluster miniMRYarnCluster) { + try { + if (miniMRYarnCluster != null) + miniMRYarnCluster.stop(); + + } catch (Exception ignored) { + // nothing we can do + ignored.printStackTrace(); + } + } + + private void closeDfs(DistributedFileSystem dfs) { + try { + if (dfs != null) + dfs.close(); + + } catch (Exception ignored) { + // nothing we can do + ignored.printStackTrace(); + } + } + + private void closeClient(JobClient client) { + try { + if (client != null) + client.close(); + } catch (Exception ignored) { + // nothing we can do + ignored.printStackTrace(); } } } @@ -155,32 +180,27 @@ class DummySocketFactory extends Standar public DummySocketFactory() { } - /* @inheritDoc */ @Override public Socket createSocket() throws IOException { return new Socket() { @Override - public void connect(SocketAddress addr, int timeout) - throws IOException { + public void connect(SocketAddress addr, int timeout) throws IOException { assert (addr instanceof InetSocketAddress); InetSocketAddress iaddr = (InetSocketAddress) addr; SocketAddress newAddr = null; if (iaddr.isUnresolved()) - newAddr = - new InetSocketAddress(iaddr.getHostName(), - iaddr.getPort() - 10); + newAddr = new InetSocketAddress(iaddr.getHostName(), + iaddr.getPort() - 10); else - newAddr = - new InetSocketAddress(iaddr.getAddress(), iaddr.getPort() - 10); - System.out.printf("Test socket: rerouting %s to %s\n", iaddr, - newAddr); + newAddr = new InetSocketAddress(iaddr.getAddress(), + iaddr.getPort() - 10); + System.out.printf("Test socket: rerouting %s to %s\n", iaddr, newAddr); super.connect(newAddr, timeout); } }; } - /* @inheritDoc */ @Override public boolean equals(Object obj) { if (this == obj) @@ -191,11 +211,4 @@ class DummySocketFactory extends Standar return false; return true; } - - /* @inheritDoc */ - @Override - public int hashCode() { - // Dummy hash code (to make find bugs happy) - return 53; - } } Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetDelegationTokenRequest.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetDelegationTokenRequest.java?rev=1327724&r1=1327723&r2=1327724&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetDelegationTokenRequest.java (original) +++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetDelegationTokenRequest.java Wed Apr 18 23:35:30 2012 @@ -21,11 +21,9 @@ package org.apache.hadoop.yarn.api.proto import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Evolving; -import clover.org.apache.velocity.runtime.resource.ResourceManager; - /** * The request issued by the client to get a delegation token from - * the {@link ResourceManager}. + * the {@code ResourceManager}. * for more information. */ @Public Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java?rev=1327724&r1=1327723&r2=1327724&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java (original) +++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java Wed Apr 18 23:35:30 2012 @@ -742,9 +742,10 @@ public class ApplicationMaster { */ private AMRMProtocol connectToRM() { YarnConfiguration yarnConf = new YarnConfiguration(conf); - InetSocketAddress rmAddress = NetUtils.createSocketAddr(yarnConf.get( + InetSocketAddress rmAddress = yarnConf.getSocketAddr( YarnConfiguration.RM_SCHEDULER_ADDRESS, - YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS)); + YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS, + YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT); LOG.info("Connecting to ResourceManager at " + rmAddress); return ((AMRMProtocol) rpc.getProxy(AMRMProtocol.class, rmAddress, conf)); } Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java?rev=1327724&r1=1327723&r2=1327724&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java (original) +++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java Wed Apr 18 23:35:30 2012 @@ -42,8 +42,6 @@ import org.apache.hadoop.conf.Configurat import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.security.SecurityInfo; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.ClientRMProtocol; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; @@ -723,9 +721,10 @@ public class Client { }); */ YarnConfiguration yarnConf = new YarnConfiguration(conf); - InetSocketAddress rmAddress = NetUtils.createSocketAddr(yarnConf.get( + InetSocketAddress rmAddress = yarnConf.getSocketAddr( YarnConfiguration.RM_ADDRESS, - YarnConfiguration.DEFAULT_RM_ADDRESS)); + YarnConfiguration.DEFAULT_RM_ADDRESS, + YarnConfiguration.DEFAULT_RM_PORT); LOG.info("Connecting to ResourceManager at " + rmAddress); applicationsManager = ((ClientRMProtocol) rpc.getProxy( ClientRMProtocol.class, rmAddress, conf)); Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/AMRMProtocolPBClientImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/AMRMProtocolPBClientImpl.java?rev=1327724&r1=1327723&r2=1327724&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/AMRMProtocolPBClientImpl.java (original) +++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/AMRMProtocolPBClientImpl.java Wed Apr 18 23:35:30 2012 @@ -56,7 +56,12 @@ public class AMRMProtocolPBClientImpl im AMRMProtocolPB.class, clientVersion, addr, conf); } - + public void close() { + if (this.proxy != null) { + RPC.stopProxy(this.proxy); + } + } + @Override public AllocateResponse allocate(AllocateRequest request) throws YarnRemoteException { Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1327724&r1=1327723&r2=1327724&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original) +++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Wed Apr 18 23:35:30 2012 @@ -246,6 +246,12 @@ public class YarnConfiguration extends C public static final String DEFAULT_RM_METRICS_RUNTIME_BUCKETS = "60,300,1440"; + public static final String RM_APP_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = RM_PREFIX + + "application-tokens.master-key-rolling-interval-secs"; + + public static final long DEFAULT_RM_APP_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = + 24 * 60 * 60; + //////////////////////////////// // Node Manager Configs //////////////////////////////// @@ -552,17 +558,16 @@ public class YarnConfiguration extends C } public static String getRMWebAppHostAndPort(Configuration conf) { - String addr = conf.get(YarnConfiguration.RM_WEBAPP_ADDRESS, - YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS); - Iterator it = ADDR_SPLITTER.split(addr).iterator(); - it.next(); // ignore the bind host - String port = it.next(); + int port = conf.getSocketAddr( + YarnConfiguration.RM_WEBAPP_ADDRESS, + YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, + YarnConfiguration.DEFAULT_RM_WEBAPP_PORT).getPort(); // Use apps manager address to figure out the host for webapp - addr = conf.get(YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS); - String host = ADDR_SPLITTER.split(addr).iterator().next(); - String rmAddress = JOINER.join(host, ":", port); - InetSocketAddress address = NetUtils.createSocketAddr( - rmAddress, DEFAULT_RM_WEBAPP_PORT, RM_WEBAPP_ADDRESS); + String host = conf.getSocketAddr( + YarnConfiguration.RM_ADDRESS, + YarnConfiguration.DEFAULT_RM_ADDRESS, + YarnConfiguration.DEFAULT_RM_PORT).getHostName(); + InetSocketAddress address = NetUtils.createSocketAddrForHost(host, port); StringBuffer sb = new StringBuffer(); InetAddress resolved = address.getAddress(); if (resolved == null || resolved.isAnyLocalAddress() || Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ApplicationTokenIdentifier.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ApplicationTokenIdentifier.java?rev=1327724&r1=1327723&r2=1327724&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ApplicationTokenIdentifier.java (original) +++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ApplicationTokenIdentifier.java Wed Apr 18 23:35:30 2012 @@ -23,34 +23,55 @@ import java.io.DataOutput; import java.io.IOException; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.util.BuilderUtils; +/** + * ApplicationTokenIdentifier is the TokenIdentifier to be used by + * ApplicationMasters to authenticate to the ResourceManager. + */ public class ApplicationTokenIdentifier extends TokenIdentifier { public static final Text KIND_NAME = new Text("YARN_APPLICATION_TOKEN"); - private String applicationAttemptId; + private ApplicationAttemptId applicationAttemptId; public ApplicationTokenIdentifier() { } public ApplicationTokenIdentifier(ApplicationAttemptId appAttemptId) { this(); - this.applicationAttemptId = appAttemptId.toString(); + this.applicationAttemptId = appAttemptId; + } + + @Private + public ApplicationAttemptId getApplicationAttemptId() { + return this.applicationAttemptId; } @Override public void write(DataOutput out) throws IOException { - Text.writeString(out, this.applicationAttemptId); + ApplicationId appId = this.applicationAttemptId.getApplicationId(); + out.writeLong(appId.getClusterTimestamp()); + out.writeInt(appId.getId()); + out.writeInt(this.applicationAttemptId.getAttemptId()); } @Override public void readFields(DataInput in) throws IOException { - this.applicationAttemptId = Text.readString(in); + long clusterTimeStamp = in.readLong(); + int appId = in.readInt(); + int attemptId = in.readInt(); + ApplicationId applicationId = + BuilderUtils.newApplicationId(clusterTimeStamp, appId); + this.applicationAttemptId = + BuilderUtils.newApplicationAttemptId(applicationId, attemptId); } @Override @@ -68,6 +89,7 @@ public class ApplicationTokenIdentifier .toString()); } + // TODO: Needed? @InterfaceAudience.Private public static class Renewer extends Token.TrivialRenewer { @Override Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1327724&r1=1327723&r2=1327724&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml (original) +++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml Wed Apr 18 23:35:30 2012 @@ -215,6 +215,14 @@ 30000 + + Interval for the roll over for the master key used to generate + application tokens + + yarn.resourcemanager.application-tokens.master-key-rolling-interval-secs + 86400 + + address of node manager IPC. Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java?rev=1327724&r1=1327723&r2=1327724&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java (original) +++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java Wed Apr 18 23:35:30 2012 @@ -32,7 +32,6 @@ import org.apache.avro.AvroRuntimeExcept import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.YarnException; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -75,7 +74,7 @@ public class NodeStatusUpdaterImpl exten private ContainerTokenSecretManager containerTokenSecretManager; private long heartBeatInterval; private ResourceTracker resourceTracker; - private String rmAddress; + private InetSocketAddress rmAddress; private Resource totalResource; private int httpPort; private byte[] secretKeyBytes = new byte[0]; @@ -106,9 +105,10 @@ public class NodeStatusUpdaterImpl exten @Override public synchronized void init(Configuration conf) { - this.rmAddress = - conf.get(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, - YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS); + this.rmAddress = conf.getSocketAddr( + YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS, + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT); this.heartBeatInterval = conf.getLong(YarnConfiguration.NM_TO_RM_HEARTBEAT_INTERVAL_MS, YarnConfiguration.DEFAULT_NM_TO_RM_HEARTBEAT_INTERVAL_MS); @@ -132,13 +132,10 @@ public class NodeStatusUpdaterImpl exten // NodeManager is the last service to start, so NodeId is available. this.nodeId = this.context.getNodeId(); - String httpBindAddressStr = - getConfig().get(YarnConfiguration.NM_WEBAPP_ADDRESS, - YarnConfiguration.DEFAULT_NM_WEBAPP_ADDRESS); - InetSocketAddress httpBindAddress = - NetUtils.createSocketAddr(httpBindAddressStr, - YarnConfiguration.DEFAULT_NM_WEBAPP_PORT, - YarnConfiguration.NM_WEBAPP_ADDRESS); + InetSocketAddress httpBindAddress = getConfig().getSocketAddr( + YarnConfiguration.NM_WEBAPP_ADDRESS, + YarnConfiguration.DEFAULT_NM_WEBAPP_ADDRESS, + YarnConfiguration.DEFAULT_NM_WEBAPP_PORT); try { // this.hostName = InetAddress.getLocalHost().getCanonicalHostName(); this.httpPort = httpBindAddress.getPort(); @@ -178,9 +175,6 @@ public class NodeStatusUpdaterImpl exten protected ResourceTracker getRMClient() { Configuration conf = getConfig(); YarnRPC rpc = YarnRPC.create(conf); - InetSocketAddress rmAddress = NetUtils.createSocketAddr(this.rmAddress, - YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT, - YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS); return (ResourceTracker) rpc.getProxy(ResourceTracker.class, rmAddress, conf); } Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java?rev=1327724&r1=1327723&r2=1327724&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java (original) +++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java Wed Apr 18 23:35:30 2012 @@ -33,7 +33,6 @@ import org.apache.hadoop.conf.Configurat import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.io.DataInputByteBuffer; import org.apache.hadoop.ipc.Server; -import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.PolicyProvider; @@ -226,10 +225,10 @@ public class ContainerManagerImpl extend Configuration conf = getConfig(); YarnRPC rpc = YarnRPC.create(conf); - InetSocketAddress initialAddress = NetUtils.createSocketAddr(conf.get( - YarnConfiguration.NM_ADDRESS, YarnConfiguration.DEFAULT_NM_ADDRESS), - YarnConfiguration.DEFAULT_NM_PORT, - YarnConfiguration.NM_ADDRESS); + InetSocketAddress initialAddress = conf.getSocketAddr( + YarnConfiguration.NM_ADDRESS, + YarnConfiguration.DEFAULT_NM_ADDRESS, + YarnConfiguration.DEFAULT_NM_PORT); server = rpc.getServer(ContainerManager.class, this, initialAddress, conf, Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java?rev=1327724&r1=1327723&r2=1327724&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java (original) +++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java Wed Apr 18 23:35:30 2012 @@ -207,10 +207,10 @@ public class ResourceLocalizationService conf.getLong(YarnConfiguration.NM_LOCALIZER_CACHE_TARGET_SIZE_MB, YarnConfiguration.DEFAULT_NM_LOCALIZER_CACHE_TARGET_SIZE_MB) << 20; cacheCleanupPeriod = conf.getLong(YarnConfiguration.NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS, YarnConfiguration.DEFAULT_NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS); - localizationServerAddress = NetUtils.createSocketAddr( - conf.get(YarnConfiguration.NM_LOCALIZER_ADDRESS, YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS), - YarnConfiguration.DEFAULT_NM_LOCALIZER_PORT, - YarnConfiguration.NM_LOCALIZER_ADDRESS); + localizationServerAddress = conf.getSocketAddr( + YarnConfiguration.NM_LOCALIZER_ADDRESS, + YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS, + YarnConfiguration.DEFAULT_NM_LOCALIZER_PORT); localizerTracker = createLocalizerTracker(conf); addService(localizerTracker); dispatcher.register(LocalizerEventType.class, localizerTracker); @@ -232,9 +232,10 @@ public class ResourceLocalizationService .split(":")[0]; getConfig().set(YarnConfiguration.NM_LOCALIZER_ADDRESS, host + ":" + server.getPort()); - localizationServerAddress = NetUtils.createSocketAddr( - getConfig().get(YarnConfiguration.NM_LOCALIZER_ADDRESS, - YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS)); + localizationServerAddress = getConfig().getSocketAddr( + YarnConfiguration.NM_LOCALIZER_ADDRESS, + YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS, + YarnConfiguration.DEFAULT_NM_LOCALIZER_PORT); LOG.info("Localizer started on port " + server.getPort()); super.start(); } Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/security/LocalizerSecurityInfo.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/security/LocalizerSecurityInfo.java?rev=1327724&r1=1327723&r2=1327724&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/security/LocalizerSecurityInfo.java (original) +++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/security/LocalizerSecurityInfo.java Wed Apr 18 23:35:30 2012 @@ -20,6 +20,8 @@ package org.apache.hadoop.yarn.server.no import java.lang.annotation.Annotation; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.KerberosInfo; import org.apache.hadoop.security.SecurityInfo; @@ -30,6 +32,8 @@ import org.apache.hadoop.yarn.server.nod public class LocalizerSecurityInfo extends SecurityInfo { + private static final Log LOG = LogFactory.getLog(LocalizerSecurityInfo.class); + @Override public KerberosInfo getKerberosInfo(Class protocol, Configuration conf) { return null; @@ -51,7 +55,7 @@ public class LocalizerSecurityInfo exten @Override public Class> value() { - System.err.print("=========== Using localizerTokenSecurityInfo"); + LOG.debug("Using localizerTokenSecurityInfo"); return LocalizerTokenSelector.class; } }; Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java?rev=1327724&r1=1327723&r2=1327724&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java (original) +++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java Wed Apr 18 23:35:30 2012 @@ -26,7 +26,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; -import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.Groups; import org.apache.hadoop.security.UserGroupInformation; @@ -95,12 +94,10 @@ public class AdminService extends Abstra @Override public void init(Configuration conf) { super.init(conf); - String bindAddress = - conf.get(YarnConfiguration.RM_ADMIN_ADDRESS, - YarnConfiguration.DEFAULT_RM_ADMIN_ADDRESS); - masterServiceAddress = NetUtils.createSocketAddr(bindAddress, - YarnConfiguration.DEFAULT_RM_ADMIN_PORT, - YarnConfiguration.RM_ADMIN_ADDRESS); + masterServiceAddress = conf.getSocketAddr( + YarnConfiguration.RM_ADMIN_ADDRESS, + YarnConfiguration.DEFAULT_RM_ADMIN_ADDRESS, + YarnConfiguration.DEFAULT_RM_ADMIN_PORT); adminAcl = new AccessControlList(conf.get( YarnConfiguration.YARN_ADMIN_ACL, YarnConfiguration.DEFAULT_YARN_ADMIN_ACL)); Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java?rev=1327724&r1=1327723&r2=1327724&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java (original) +++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java Wed Apr 18 23:35:30 2012 @@ -55,7 +55,6 @@ import org.apache.hadoop.yarn.factories. import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.RPCUtil; import org.apache.hadoop.yarn.ipc.YarnRPC; -import org.apache.hadoop.yarn.security.ApplicationTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeState; @@ -72,14 +71,14 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.service.AbstractService; import org.apache.hadoop.yarn.util.BuilderUtils; +@SuppressWarnings("unchecked") @Private public class ApplicationMasterService extends AbstractService implements AMRMProtocol { private static final Log LOG = LogFactory.getLog(ApplicationMasterService.class); private final AMLivelinessMonitor amLivelinessMonitor; private YarnScheduler rScheduler; - private ApplicationTokenSecretManager appTokenManager; - private InetSocketAddress masterServiceAddress; + private InetSocketAddress bindAddress; private Server server; private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); private final ConcurrentMap responseMap = @@ -87,11 +86,9 @@ public class ApplicationMasterService ex private final AMResponse reboot = recordFactory.newRecordInstance(AMResponse.class); private final RMContext rmContext; - public ApplicationMasterService(RMContext rmContext, - ApplicationTokenSecretManager appTokenManager, YarnScheduler scheduler) { + public ApplicationMasterService(RMContext rmContext, YarnScheduler scheduler) { super(ApplicationMasterService.class.getName()); this.amLivelinessMonitor = rmContext.getAMLivelinessMonitor(); - this.appTokenManager = appTokenManager; this.rScheduler = scheduler; this.reboot.setReboot(true); // this.reboot.containers = new ArrayList(); @@ -99,23 +96,18 @@ public class ApplicationMasterService ex } @Override - public void init(Configuration conf) { - String bindAddress = - conf.get(YarnConfiguration.RM_SCHEDULER_ADDRESS, - YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS); - masterServiceAddress = NetUtils.createSocketAddr(bindAddress, - YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT, - YarnConfiguration.RM_SCHEDULER_ADDRESS); - super.init(conf); - } - - @Override public void start() { Configuration conf = getConfig(); YarnRPC rpc = YarnRPC.create(conf); + + InetSocketAddress masterServiceAddress = conf.getSocketAddr( + YarnConfiguration.RM_SCHEDULER_ADDRESS, + YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS, + YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT); + this.server = rpc.getServer(AMRMProtocol.class, this, masterServiceAddress, - conf, this.appTokenManager, + conf, this.rmContext.getApplicationTokenSecretManager(), conf.getInt(YarnConfiguration.RM_SCHEDULER_CLIENT_THREAD_COUNT, YarnConfiguration.DEFAULT_RM_SCHEDULER_CLIENT_THREAD_COUNT)); @@ -127,9 +119,19 @@ public class ApplicationMasterService ex } this.server.start(); + + this.bindAddress = + NetUtils.createSocketAddr(masterServiceAddress.getHostName(), + this.server.getPort()); + super.start(); } + @Private + public InetSocketAddress getBindAddress() { + return this.bindAddress; + } + private void authorizeRequest(ApplicationAttemptId appAttemptID) throws YarnRemoteException { Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java?rev=1327724&r1=1327723&r2=1327724&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java (original) +++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java Wed Apr 18 23:35:30 2012 @@ -33,7 +33,6 @@ import org.apache.hadoop.conf.Configurat import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.io.Text; import org.apache.hadoop.ipc.Server; -import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; import org.apache.hadoop.security.authorize.PolicyProvider; @@ -104,7 +103,6 @@ public class ClientRMService extends Abs final private RMContext rmContext; private final RMAppManager rmAppManager; - private String clientServiceBindAddress; private Server server; private RMDelegationTokenSecretManager rmDTSecretManager; @@ -126,13 +124,10 @@ public class ClientRMService extends Abs @Override public void init(Configuration conf) { - clientServiceBindAddress = - conf.get(YarnConfiguration.RM_ADDRESS, - YarnConfiguration.DEFAULT_RM_ADDRESS); - clientBindAddress = - NetUtils.createSocketAddr(clientServiceBindAddress, - YarnConfiguration.DEFAULT_RM_PORT, - YarnConfiguration.RM_ADDRESS); + clientBindAddress = conf.getSocketAddr( + YarnConfiguration.RM_ADDRESS, + YarnConfiguration.DEFAULT_RM_ADDRESS, + YarnConfiguration.DEFAULT_RM_PORT); super.init(conf); } Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java?rev=1327724&r1=1327723&r2=1327724&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java (original) +++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java Wed Apr 18 23:35:30 2012 @@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; +import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer; /** @@ -53,4 +54,6 @@ public interface RMContext { ContainerAllocationExpirer getContainerAllocationExpirer(); DelegationTokenRenewer getDelegationTokenRenewer(); + + ApplicationTokenSecretManager getApplicationTokenSecretManager(); } \ No newline at end of file Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java?rev=1327724&r1=1327723&r2=1327724&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java (original) +++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java Wed Apr 18 23:35:30 2012 @@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; +import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer; public class RMContextImpl implements RMContext { @@ -50,16 +51,19 @@ public class RMContextImpl implements RM private AMLivelinessMonitor amLivelinessMonitor; private ContainerAllocationExpirer containerAllocationExpirer; private final DelegationTokenRenewer tokenRenewer; + private final ApplicationTokenSecretManager appTokenSecretManager; public RMContextImpl(Store store, Dispatcher rmDispatcher, ContainerAllocationExpirer containerAllocationExpirer, AMLivelinessMonitor amLivelinessMonitor, - DelegationTokenRenewer tokenRenewer) { + DelegationTokenRenewer tokenRenewer, + ApplicationTokenSecretManager appTokenSecretManager) { this.store = store; this.rmDispatcher = rmDispatcher; this.containerAllocationExpirer = containerAllocationExpirer; this.amLivelinessMonitor = amLivelinessMonitor; this.tokenRenewer = tokenRenewer; + this.appTokenSecretManager = appTokenSecretManager; } @Override @@ -106,4 +110,9 @@ public class RMContextImpl implements RM public DelegationTokenRenewer getDelegationTokenRenewer() { return tokenRenewer; } + + @Override + public ApplicationTokenSecretManager getApplicationTokenSecretManager() { + return this.appTokenSecretManager; + } } \ No newline at end of file Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java?rev=1327724&r1=1327723&r2=1327724&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java (original) +++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java Wed Apr 18 23:35:30 2012 @@ -27,7 +27,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.metrics2.MetricsSystem; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.metrics2.source.JvmMetrics; import org.apache.hadoop.security.SecurityUtil; @@ -41,7 +40,6 @@ import org.apache.hadoop.yarn.conf.YarnC import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.EventHandler; -import org.apache.hadoop.yarn.security.ApplicationTokenSecretManager; import org.apache.hadoop.yarn.security.client.ClientToAMSecretManager; import org.apache.hadoop.yarn.server.RMDelegationTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEventType; @@ -65,6 +63,7 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer; import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebApp; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; @@ -82,8 +81,10 @@ import org.apache.hadoop.yarn.webapp.Web /** * The ResourceManager is the main class that is a set of components. + * "I am the ResourceManager. All your resources are belong to us..." * */ +@SuppressWarnings("unchecked") public class ResourceManager extends CompositeService implements Recoverable { private static final Log LOG = LogFactory.getLog(ResourceManager.class); public static final long clusterTimeStamp = System.currentTimeMillis(); @@ -94,8 +95,7 @@ public class ResourceManager extends Com protected ContainerTokenSecretManager containerTokenSecretManager = new ContainerTokenSecretManager(); - protected ApplicationTokenSecretManager appTokenSecretManager = - new ApplicationTokenSecretManager(); + protected ApplicationTokenSecretManager appTokenSecretManager; private Dispatcher rmDispatcher; @@ -137,6 +137,8 @@ public class ResourceManager extends Com this.rmDispatcher = createDispatcher(); addIfService(this.rmDispatcher); + this.appTokenSecretManager = createApplicationTokenSecretManager(conf); + this.containerAllocationExpirer = new ContainerAllocationExpirer( this.rmDispatcher); addService(this.containerAllocationExpirer); @@ -147,8 +149,10 @@ public class ResourceManager extends Com DelegationTokenRenewer tokenRenewer = createDelegationTokenRenewer(); addService(tokenRenewer); - this.rmContext = new RMContextImpl(this.store, this.rmDispatcher, - this.containerAllocationExpirer, amLivelinessMonitor, tokenRenewer); + this.rmContext = + new RMContextImpl(this.store, this.rmDispatcher, + this.containerAllocationExpirer, amLivelinessMonitor, tokenRenewer, + this.appTokenSecretManager); // Register event handler for NodesListManager this.nodesListManager = new NodesListManager(this.rmContext); @@ -175,10 +179,6 @@ public class ResourceManager extends Com this.rmDispatcher.register(RMNodeEventType.class, new NodeEventDispatcher(this.rmContext)); - //TODO change this to be random - this.appTokenSecretManager.setMasterKey(ApplicationTokenSecretManager - .createSecretKey("Dummy".getBytes())); - this.nmLivelinessMonitor = createNMLivelinessMonitor(); addService(this.nmLivelinessMonitor); @@ -233,6 +233,11 @@ public class ResourceManager extends Com } } + protected ApplicationTokenSecretManager createApplicationTokenSecretManager( + Configuration conf) { + return new ApplicationTokenSecretManager(conf); + } + protected ResourceScheduler createScheduler() { return ReflectionUtils.newInstance(this.conf.getClass( YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, @@ -240,9 +245,8 @@ public class ResourceManager extends Com } protected ApplicationMasterLauncher createAMLauncher() { - return new ApplicationMasterLauncher( - this.appTokenSecretManager, this.clientToAMSecretManager, - this.rmContext); + return new ApplicationMasterLauncher(this.clientToAMSecretManager, + this.rmContext); } private NMLivelinessMonitor createNMLivelinessMonitor() { @@ -273,6 +277,7 @@ public class ResourceManager extends Com new LinkedBlockingQueue(); private final Thread eventProcessor; private volatile boolean stopped = false; + private boolean shouldExitOnError = false; public SchedulerEventDispatcher(ResourceScheduler scheduler) { super(SchedulerEventDispatcher.class.getName()); @@ -282,6 +287,14 @@ public class ResourceManager extends Com } @Override + public synchronized void init(Configuration conf) { + this.shouldExitOnError = + conf.getBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, + Dispatcher.DEFAULT_DISPATCHER_EXIT_ON_ERROR); + super.init(conf); + } + + @Override public synchronized void start() { this.eventProcessor.start(); super.start(); @@ -306,8 +319,7 @@ public class ResourceManager extends Com } catch (Throwable t) { LOG.fatal("Error in handling event type " + event.getType() + " to the scheduler", t); - if (getConfig().getBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, - Dispatcher.DEFAULT_DISPATCHER_EXIT_ON_ERROR)) { + if (shouldExitOnError) { LOG.info("Exiting, bbye.."); System.exit(-1); } @@ -453,6 +465,8 @@ public class ResourceManager extends Com throw new YarnException("Failed to login", ie); } + this.appTokenSecretManager.start(); + startWepApp(); DefaultMetricsSystem.initialize("ResourceManager"); JvmMetrics.initSingleton("ResourceManager", null); @@ -487,6 +501,8 @@ public class ResourceManager extends Com } rmDTSecretManager.stopThreads(); + this.appTokenSecretManager.stop(); + /*synchronized(shutdown) { shutdown.set(true); shutdown.notifyAll(); @@ -524,8 +540,7 @@ public class ResourceManager extends Com } protected ApplicationMasterService createApplicationMasterService() { - return new ApplicationMasterService(this.rmContext, - this.appTokenSecretManager, scheduler); + return new ApplicationMasterService(this.rmContext, scheduler); } @@ -571,6 +586,11 @@ public class ResourceManager extends Com return this.applicationACLsManager; } + @Private + public ApplicationTokenSecretManager getApplicationTokenSecretManager(){ + return this.appTokenSecretManager; + } + @Override public void recover(RMState state) throws Exception { resourceTracker.recover(state); Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java?rev=1327724&r1=1327723&r2=1327724&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java (original) +++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java Wed Apr 18 23:35:30 2012 @@ -27,7 +27,6 @@ import org.apache.commons.logging.LogFac import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.ipc.Server; -import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.net.Node; import org.apache.hadoop.security.authorize.PolicyProvider; import org.apache.hadoop.yarn.api.records.NodeId; @@ -104,13 +103,10 @@ public class ResourceTrackerService exte @Override public synchronized void init(Configuration conf) { - String resourceTrackerBindAddress = - conf.get(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, - YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS); - resourceTrackerAddress = NetUtils.createSocketAddr( - resourceTrackerBindAddress, - YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT, - YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS); + resourceTrackerAddress = conf.getSocketAddr( + YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS, + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT); RackResolver.init(conf); super.init(conf); Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java?rev=1327724&r1=1327723&r2=1327724&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java (original) +++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java Wed Apr 18 23:35:30 2012 @@ -53,7 +53,6 @@ import org.apache.hadoop.yarn.factories. import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.security.ApplicationTokenIdentifier; -import org.apache.hadoop.yarn.security.ApplicationTokenSecretManager; import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.security.client.ClientToAMSecretManager; import org.apache.hadoop.yarn.security.client.ClientTokenIdentifier; @@ -76,7 +75,6 @@ public class AMLauncher implements Runna private final Configuration conf; private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); - private final ApplicationTokenSecretManager applicationTokenSecretManager; private final ClientToAMSecretManager clientToAMSecretManager; private final AMLauncherEventType eventType; private final RMContext rmContext; @@ -86,11 +84,9 @@ public class AMLauncher implements Runna public AMLauncher(RMContext rmContext, RMAppAttempt application, AMLauncherEventType eventType, - ApplicationTokenSecretManager applicationTokenSecretManager, ClientToAMSecretManager clientToAMSecretManager, Configuration conf) { this.application = application; this.conf = conf; - this.applicationTokenSecretManager = applicationTokenSecretManager; this.clientToAMSecretManager = clientToAMSecretManager; this.eventType = eventType; this.rmContext = rmContext; @@ -129,6 +125,7 @@ public class AMLauncher implements Runna containerMgrProxy.stopContainer(stopRequest); } + // Protected. For tests. protected ContainerManager getContainerMgrProxy( final ContainerId containerId) { @@ -220,12 +217,11 @@ public class AMLauncher implements Runna application.getAppAttemptId()); Token token = new Token(id, - this.applicationTokenSecretManager); - String schedulerAddressStr = - this.conf.get(YarnConfiguration.RM_SCHEDULER_ADDRESS, - YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS); - InetSocketAddress unresolvedAddr = - NetUtils.createSocketAddr(schedulerAddressStr); + this.rmContext.getApplicationTokenSecretManager()); + InetSocketAddress unresolvedAddr = conf.getSocketAddr( + YarnConfiguration.RM_SCHEDULER_ADDRESS, + YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS, + YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT); String resolvedAddr = unresolvedAddr.getAddress().getHostAddress() + ":" + unresolvedAddr.getPort(); Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/ApplicationMasterLauncher.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/ApplicationMasterLauncher.java?rev=1327724&r1=1327723&r2=1327724&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/ApplicationMasterLauncher.java (original) +++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/ApplicationMasterLauncher.java Wed Apr 18 23:35:30 2012 @@ -25,7 +25,6 @@ import java.util.concurrent.TimeUnit; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.yarn.event.EventHandler; -import org.apache.hadoop.yarn.security.ApplicationTokenSecretManager; import org.apache.hadoop.yarn.security.client.ClientToAMSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; @@ -42,20 +41,16 @@ public class ApplicationMasterLauncher e private final BlockingQueue masterEvents = new LinkedBlockingQueue(); - protected ApplicationTokenSecretManager applicationTokenSecretManager; private ClientToAMSecretManager clientToAMSecretManager; protected final RMContext context; public ApplicationMasterLauncher( - ApplicationTokenSecretManager applicationTokenSecretManager, - ClientToAMSecretManager clientToAMSecretManager, - RMContext context) { + ClientToAMSecretManager clientToAMSecretManager, RMContext context) { super(ApplicationMasterLauncher.class.getName()); this.context = context; this.launcherPool = new ThreadPoolExecutor(10, 10, 1, TimeUnit.HOURS, new LinkedBlockingQueue()); this.launcherHandlingThread = new LauncherThread(); - this.applicationTokenSecretManager = applicationTokenSecretManager; this.clientToAMSecretManager = clientToAMSecretManager; } @@ -66,8 +61,9 @@ public class ApplicationMasterLauncher e protected Runnable createRunnableLauncher(RMAppAttempt application, AMLauncherEventType event) { - Runnable launcher = new AMLauncher(context, application, event, - applicationTokenSecretManager, clientToAMSecretManager, getConfig()); + Runnable launcher = + new AMLauncher(context, application, event, clientToAMSecretManager, + getConfig()); return launcher; } Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java?rev=1327724&r1=1327723&r2=1327724&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java (original) +++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java Wed Apr 18 23:35:30 2012 @@ -33,13 +33,13 @@ import org.apache.commons.logging.LogFac import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.YarnException; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; -import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; -import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.EventHandler; Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java?rev=1327724&r1=1327723&r2=1327724&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java (original) +++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java Wed Apr 18 23:35:30 2012 @@ -77,6 +77,7 @@ import org.apache.hadoop.yarn.state.Stat import org.apache.hadoop.yarn.state.StateMachineFactory; import org.apache.hadoop.yarn.util.BuilderUtils; +@SuppressWarnings({"unchecked", "rawtypes"}) public class RMAppAttemptImpl implements RMAppAttempt { private static final Log LOG = LogFactory.getLog(RMAppAttemptImpl.class); @@ -95,7 +96,6 @@ public class RMAppAttemptImpl implements RMAppAttemptEvent> stateMachine; private final RMContext rmContext; - @SuppressWarnings("rawtypes") private final EventHandler eventHandler; private final YarnScheduler scheduler; private final ApplicationMasterService masterService; @@ -539,7 +539,6 @@ public class RMAppAttemptImpl implements } private static final class AttemptStartedTransition extends BaseTransition { - @SuppressWarnings("unchecked") @Override public void transition(RMAppAttemptImpl appAttempt, RMAppAttemptEvent event) { @@ -638,12 +637,13 @@ public class RMAppAttemptImpl implements public void transition(RMAppAttemptImpl appAttempt, RMAppAttemptEvent event) { + ApplicationAttemptId appAttemptId = appAttempt.getAppAttemptId(); + // Tell the AMS. Unregister from the ApplicationMasterService - appAttempt.masterService - .unregisterAttempt(appAttempt.applicationAttemptId); + appAttempt.masterService.unregisterAttempt(appAttemptId); // Tell the application and the scheduler - ApplicationId applicationId = appAttempt.getAppAttemptId().getApplicationId(); + ApplicationId applicationId = appAttemptId.getApplicationId(); RMAppEvent appEvent = null; switch (finalAttemptState) { case FINISHED: @@ -676,8 +676,12 @@ public class RMAppAttemptImpl implements } appAttempt.eventHandler.handle(appEvent); - appAttempt.eventHandler.handle(new AppRemovedSchedulerEvent(appAttempt - .getAppAttemptId(), finalAttemptState)); + appAttempt.eventHandler.handle(new AppRemovedSchedulerEvent(appAttemptId, + finalAttemptState)); + + // Remove the AppAttempt from the ApplicationTokenSecretManager + appAttempt.rmContext.getApplicationTokenSecretManager() + .applicationMasterFinished(appAttemptId); } } Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1327724&r1=1327723&r2=1327724&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (original) +++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java Wed Apr 18 23:35:30 2012 @@ -1118,13 +1118,12 @@ public class LeafQueue implements CSQueu boolean canAssign(SchedulerApp application, Priority priority, SchedulerNode node, NodeType type, RMContainer reservedContainer) { - // Reserved... - if (reservedContainer != null) { - return true; - } - // Clearly we need containers for this application... if (type == NodeType.OFF_SWITCH) { + if (reservedContainer != null) { + return true; + } + // 'Delay' off-switch ResourceRequest offSwitchRequest = application.getResourceRequest(priority, RMNode.ANY); Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java?rev=1327724&r1=1327723&r2=1327724&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java (original) +++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java Wed Apr 18 23:35:30 2012 @@ -185,8 +185,6 @@ public class FifoScheduler implements Re @Override public synchronized void setConf(Configuration conf) { this.conf = conf; - metrics = QueueMetrics.forQueue(DEFAULT_QUEUE_NAME, null, false, conf); - activeUsersManager = new ActiveUsersManager(metrics); } @Override @@ -223,6 +221,9 @@ public class FifoScheduler implements Re Resources.createResource(conf.getInt(MINIMUM_ALLOCATION, MINIMUM_MEMORY)); this.maximumAllocation = Resources.createResource(conf.getInt(MAXIMUM_ALLOCATION, MAXIMUM_MEMORY)); + this.metrics = QueueMetrics.forQueue(DEFAULT_QUEUE_NAME, null, false, + conf); + this.activeUsersManager = new ActiveUsersManager(metrics); this.initialized = true; } } Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/tools/RMAdmin.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/tools/RMAdmin.java?rev=1327724&r1=1327723&r2=1327724&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/tools/RMAdmin.java (original) +++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/tools/RMAdmin.java Wed Apr 18 23:35:30 2012 @@ -25,7 +25,6 @@ import java.security.PrivilegedAction; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.ipc.RemoteException; -import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; @@ -158,13 +157,10 @@ public class RMAdmin extends Configured final YarnConfiguration conf = new YarnConfiguration(getConf()); // Create the client - final String adminAddress = - conf.get(YarnConfiguration.RM_ADMIN_ADDRESS, - YarnConfiguration.DEFAULT_RM_ADMIN_ADDRESS); - final InetSocketAddress addr = - NetUtils.createSocketAddr(adminAddress, - YarnConfiguration.DEFAULT_RM_ADMIN_PORT, - YarnConfiguration.RM_ADMIN_ADDRESS); + final InetSocketAddress addr = conf.getSocketAddr( + YarnConfiguration.RM_ADMIN_ADDRESS, + YarnConfiguration.DEFAULT_RM_ADMIN_ADDRESS, + YarnConfiguration.DEFAULT_RM_ADMIN_PORT); final YarnRPC rpc = YarnRPC.create(conf); RMAdminProtocol adminProtocol = Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java?rev=1327724&r1=1327723&r2=1327724&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java (original) +++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java Wed Apr 18 23:35:30 2012 @@ -23,6 +23,9 @@ import static org.apache.hadoop.yarn.web import static org.apache.hadoop.yarn.webapp.view.JQueryUI._PROGRESSBAR; import static org.apache.hadoop.yarn.webapp.view.JQueryUI._PROGRESSBAR_VALUE; +import java.util.Collection; +import java.util.HashSet; + import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; @@ -60,10 +63,17 @@ class AppsBlock extends HtmlBlock { th(".ui", "Tracking UI")._()._(). tbody(); int i = 0; - String reqState = $(APP_STATE); - reqState = (reqState == null ? "" : reqState); + Collection reqAppStates = null; + String reqStateString = $(APP_STATE); + if (reqStateString != null && !reqStateString.isEmpty()) { + String[] appStateStrings = reqStateString.split(","); + reqAppStates = new HashSet(appStateStrings.length); + for(String stateString : appStateStrings) { + reqAppStates.add(RMAppState.valueOf(stateString)); + } + } for (RMApp app : list.apps.values()) { - if (!reqState.isEmpty() && app.getState() != RMAppState.valueOf(reqState)) { + if (reqAppStates != null && !reqAppStates.contains(app.getState())) { continue; } AppInfo appInfo = new AppInfo(app, true); @@ -100,7 +110,7 @@ class AppsBlock extends HtmlBlock { if (list.rendering == Render.JS_ARRAY) { echo("\n"); } } Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsList.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsList.java?rev=1327724&r1=1327723&r2=1327724&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsList.java (original) +++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsList.java Wed Apr 18 23:35:30 2012 @@ -26,6 +26,7 @@ import static org.apache.hadoop.yarn.web import static org.apache.hadoop.yarn.webapp.view.Jsons.appendSortable; import java.io.PrintWriter; +import java.util.Collection; import java.util.concurrent.ConcurrentMap; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -53,12 +54,12 @@ class AppsList implements ToJSON { apps = rmContext.getRMApps(); } - void toDataTableArrays(String requiredAppState, PrintWriter out) { + void toDataTableArrays(Collection requiredAppStates, PrintWriter out) { out.append('['); boolean first = true; for (RMApp app : apps.values()) { - if (requiredAppState != null && !requiredAppState.isEmpty() - && app.getState() != RMAppState.valueOf(requiredAppState)) { + if (requiredAppStates != null && + !requiredAppStates.contains(app.getState())) { continue; } AppInfo appInfo = new AppInfo(app, true); Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java?rev=1327724&r1=1327723&r2=1327724&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java (original) +++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java Wed Apr 18 23:35:30 2012 @@ -31,14 +31,17 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.util.Apps; +import org.apache.hadoop.yarn.util.StringHelper; import org.apache.hadoop.yarn.util.Times; import org.apache.hadoop.yarn.webapp.Controller; import org.apache.hadoop.yarn.webapp.ResponseInfo; +import org.apache.hadoop.yarn.webapp.YarnWebParams; import com.google.inject.Inject; @@ -123,6 +126,13 @@ public class RmController extends Contro } public void scheduler() { + // limit applications to those in states relevant to scheduling + set(YarnWebParams.APP_STATE, StringHelper.cjoin( + RMAppState.NEW.toString(), + RMAppState.SUBMITTED.toString(), + RMAppState.ACCEPTED.toString(), + RMAppState.RUNNING.toString())); + ResourceManager rm = getInstance(ResourceManager.class); ResourceScheduler rs = rm.getResourceScheduler(); if (rs == null || rs instanceof CapacityScheduler) { Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java?rev=1327724&r1=1327723&r2=1327724&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java (original) +++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java Wed Apr 18 23:35:30 2012 @@ -24,9 +24,9 @@ import junit.framework.Assert; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.ClientRMProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -35,7 +35,6 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEvent; import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher; import org.apache.hadoop.yarn.server.resourcemanager.recovery.StoreFactory; @@ -55,6 +54,7 @@ import org.apache.log4j.Level; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; +@SuppressWarnings("unchecked") public class MockRM extends ResourceManager { public MockRM() { @@ -224,8 +224,7 @@ public class MockRM extends ResourceMana @Override protected ApplicationMasterService createApplicationMasterService() { - return new ApplicationMasterService(getRMContext(), - this.appTokenSecretManager, scheduler) { + return new ApplicationMasterService(getRMContext(), scheduler) { @Override public void start() { // override to not start rpc handler @@ -240,8 +239,8 @@ public class MockRM extends ResourceMana @Override protected ApplicationMasterLauncher createAMLauncher() { - return new ApplicationMasterLauncher(this.appTokenSecretManager, - this.clientToAMSecretManager, getRMContext()) { + return new ApplicationMasterLauncher(this.clientToAMSecretManager, + getRMContext()) { @Override public void start() { // override to not start rpc handler