Return-Path: X-Original-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 164669DEE for ; Thu, 19 Apr 2012 20:33:36 +0000 (UTC) Received: (qmail 61051 invoked by uid 500); 19 Apr 2012 20:33:35 -0000 Delivered-To: apmail-hadoop-mapreduce-commits-archive@hadoop.apache.org Received: (qmail 60854 invoked by uid 500); 19 Apr 2012 20:33:35 -0000 Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: mapreduce-dev@hadoop.apache.org Delivered-To: mailing list mapreduce-commits@hadoop.apache.org Received: (qmail 60842 invoked by uid 99); 19 Apr 2012 20:33:35 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 19 Apr 2012 20:33:35 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 19 Apr 2012 20:33:33 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id CF7C92388B3A; Thu, 19 Apr 2012 20:33:12 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1328109 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/ hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/... Date: Thu, 19 Apr 2012 20:33:12 -0000 To: mapreduce-commits@hadoop.apache.org From: tucu@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120419203312.CF7C92388B3A@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: tucu Date: Thu Apr 19 20:33:11 2012 New Revision: 1328109 URL: http://svn.apache.org/viewvc?rev=1328109&view=rev Log: MAPREDUCE-3867. MiniMRYarn/MiniYarn uses fixed ports (tucu) Added: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/resources/yarn-site.xml Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/ipc/TestSocketFactory.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClasspath.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1328109&r1=1328108&r2=1328109&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Thu Apr 19 20:33:11 2012 @@ -253,6 +253,8 @@ Release 2.0.0 - UNRELEASED MAPREDUCE-4008. ResourceManager throws MetricsException on start up saying QueueMetrics MBean already exists (Devaraj K via tgraves) + MAPREDUCE-3867. MiniMRYarn/MiniYarn uses fixed ports (tucu) + Release 0.23.3 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java?rev=1328109&r1=1328108&r2=1328109&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java Thu Apr 19 20:33:11 2012 @@ -78,6 +78,7 @@ import org.apache.hadoop.security.UserGr import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.YarnException; import org.apache.hadoop.yarn.api.records.DelegationToken; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; @@ -147,6 +148,18 @@ public class HistoryClientService extend + ":" + server.getPort()); LOG.info("Instantiated MRClientService at " + this.bindAddress); + if (getConfig().getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) { + String resolvedAddress = bindAddress.getHostName() + ":" + bindAddress.getPort(); + conf.set(JHAdminConfig.MR_HISTORY_ADDRESS, resolvedAddress); + + String hostname = getConfig().get(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS, + JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_ADDRESS); + hostname = (hostname.contains(":")) ? hostname.substring(0, hostname.indexOf(":")) : hostname; + int port = webApp.port(); + resolvedAddress = hostname + ":" + port; + conf.set(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS, resolvedAddress); + } + super.start(); } Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/ipc/TestSocketFactory.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/ipc/TestSocketFactory.java?rev=1328109&r1=1328108&r2=1328109&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/ipc/TestSocketFactory.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/ipc/TestSocketFactory.java Thu Apr 19 20:33:11 2012 @@ -83,7 +83,9 @@ public class TestSocketFactory { JobConf jobConf = new JobConf(); FileSystem.setDefaultUri(jobConf, fs.getUri().toString()); miniMRYarnCluster = initAndStartMiniMRYarnCluster(jobConf); - JobConf jconf = new JobConf(cconf); + JobConf jconf = new JobConf(miniMRYarnCluster.getConfig()); + jconf.set("hadoop.rpc.socket.factory.class.default", + "org.apache.hadoop.ipc.DummySocketFactory"); jconf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME); String rmAddress = jconf.get("yarn.resourcemanager.address"); String[] split = rmAddress.split(":"); Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClasspath.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClasspath.java?rev=1328109&r1=1328108&r2=1328109&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClasspath.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClasspath.java Thu Apr 19 20:33:11 2012 @@ -173,7 +173,7 @@ public class TestMiniMRClasspath { fileSys = dfs.getFileSystem(); namenode = fileSys.getUri().toString(); mr = new MiniMRCluster(taskTrackers, namenode, 3); - JobConf jobConf = new JobConf(); + JobConf jobConf = mr.createJobConf(); String result; result = launchWordCount(fileSys.getUri(), jobConf, "The quick brown fox\nhas many silly\n" + "red fox sox\n", 3, 1); @@ -205,7 +205,7 @@ public class TestMiniMRClasspath { fileSys = dfs.getFileSystem(); namenode = fileSys.getUri().toString(); mr = new MiniMRCluster(taskTrackers, namenode, 3); - JobConf jobConf = new JobConf(); + JobConf jobConf = mr.createJobConf(); String result; result = launchExternal(fileSys.getUri(), jobConf, Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java?rev=1328109&r1=1328108&r2=1328109&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java Thu Apr 19 20:33:11 2012 @@ -20,6 +20,8 @@ package org.apache.hadoop.mapreduce.v2; import java.io.File; import java.io.IOException; +import java.net.InetAddress; +import java.net.UnknownHostException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -126,6 +128,10 @@ public class MiniMRYarnCluster extends M @Override public synchronized void start() { try { + getConfig().set(JHAdminConfig.MR_HISTORY_ADDRESS, + MiniYARNCluster.getHostname() + ":0"); + getConfig().set(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS, + MiniYARNCluster.getHostname() + ":0"); historyServer = new JobHistoryServer(); historyServer.init(getConfig()); new Thread() { @@ -145,6 +151,20 @@ public class MiniMRYarnCluster extends M } catch (Throwable t) { throw new YarnException(t); } + //need to do this because historyServer.init creates a new Configuration + getConfig().set(JHAdminConfig.MR_HISTORY_ADDRESS, + historyServer.getConfig().get(JHAdminConfig.MR_HISTORY_ADDRESS)); + getConfig().set(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS, + historyServer.getConfig().get(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS)); + + LOG.info("MiniMRYARN ResourceManager address: " + + getConfig().get(YarnConfiguration.RM_ADDRESS)); + LOG.info("MiniMRYARN ResourceManager web address: " + + getConfig().get(YarnConfiguration.RM_WEBAPP_ADDRESS)); + LOG.info("MiniMRYARN HistoryServer address: " + + getConfig().get(JHAdminConfig.MR_HISTORY_ADDRESS)); + LOG.info("MiniMRYARN HistoryServer web address: " + + getConfig().get(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS)); } @Override Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java?rev=1328109&r1=1328108&r2=1328109&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java Thu Apr 19 20:33:11 2012 @@ -197,13 +197,19 @@ public class Client { /** */ - public Client() throws Exception { + public Client(Configuration conf) throws Exception { // Set up the configuration and RPC - conf = new Configuration(); + this.conf = conf; rpc = YarnRPC.create(conf); } /** + */ + public Client() throws Exception { + this(new Configuration()); + } + + /** * Helper function to print out usage * @param opts Parsed command line options */ Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java?rev=1328109&r1=1328108&r2=1328109&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java Thu Apr 19 20:33:11 2012 @@ -18,11 +18,17 @@ package org.apache.hadoop.yarn.applications.distributedshell; +import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; +import java.io.OutputStream; +import java.net.URL; +import junit.framework.Assert; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.util.JarFinder; import org.apache.hadoop.yarn.server.MiniYARNCluster; import org.junit.AfterClass; @@ -48,6 +54,14 @@ public class TestDistributedShell { 1, 1, 1); yarnCluster.init(conf); yarnCluster.start(); + URL url = Thread.currentThread().getContextClassLoader().getResource("yarn-site.xml"); + if (url == null) { + throw new RuntimeException("Could not find 'yarn-site.xml' dummy file in classpath"); + } + yarnCluster.getConfig().set("yarn.application.classpath", new File(url.getPath()).getParent()); + OutputStream os = new FileOutputStream(new File(url.getPath())); + yarnCluster.getConfig().writeXml(os); + os.close(); } try { Thread.sleep(2000); @@ -81,14 +95,14 @@ public class TestDistributedShell { }; LOG.info("Initializing DS Client"); - Client client = new Client(); + Client client = new Client(new Configuration(yarnCluster.getConfig())); boolean initSuccess = client.init(args); - assert(initSuccess); + Assert.assertTrue(initSuccess); LOG.info("Running DS Client"); boolean result = client.run(); LOG.info("Client run completed. Result=" + result); - assert (result == true); + Assert.assertTrue(result); } Added: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/resources/yarn-site.xml URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/resources/yarn-site.xml?rev=1328109&view=auto ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/resources/yarn-site.xml (added) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/resources/yarn-site.xml Thu Apr 19 20:33:11 2012 @@ -0,0 +1,21 @@ + + + + + + + + + Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1328109&r1=1328108&r2=1328109&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Thu Apr 19 20:33:11 2012 @@ -538,6 +538,8 @@ public class YarnConfiguration extends C /** Container temp directory */ public static final String DEFAULT_CONTAINER_TEMP_DIR = "./tmp"; + public static final String IS_MINI_YARN_CLUSTER = YARN_PREFIX + ".is.minicluster"; + public YarnConfiguration() { super(); } Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java?rev=1328109&r1=1328108&r2=1328109&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java Thu Apr 19 20:33:11 2012 @@ -120,6 +120,11 @@ public class AdminService extends Abstra } this.server.start(); + if (getConfig().getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) { + String resolvedAddress = + this.server.getListenerAddress().getHostName() + ":" + this.server.getListenerAddress().getPort(); + conf.set(YarnConfiguration.RM_ADMIN_ADDRESS, resolvedAddress); + } super.start(); } Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java?rev=1328109&r1=1328108&r2=1328109&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java Thu Apr 19 20:33:11 2012 @@ -119,11 +119,14 @@ public class ApplicationMasterService ex } this.server.start(); - this.bindAddress = NetUtils.createSocketAddr(masterServiceAddress.getHostName(), this.server.getPort()); - + if (getConfig().getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) { + String resolvedAddress = + this.server.getListenerAddress().getHostName() + ":" + this.server.getListenerAddress().getPort(); + conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, resolvedAddress); + } super.start(); } Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java?rev=1328109&r1=1328108&r2=1328109&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java Thu Apr 19 20:33:11 2012 @@ -150,6 +150,11 @@ public class ClientRMService extends Abs } this.server.start(); + if (getConfig().getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) { + String resolvedAddress = + this.server.getListenerAddress().getHostName() + ":" + this.server.getListenerAddress().getPort(); + conf.set(YarnConfiguration.RM_ADDRESS, resolvedAddress); + } super.start(); } Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java?rev=1328109&r1=1328108&r2=1328109&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java Thu Apr 19 20:33:11 2012 @@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.re import java.io.IOException; +import java.net.InetAddress; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; @@ -475,6 +476,15 @@ public class ResourceManager extends Com } catch(IOException ie) { throw new YarnException("Failed to start secret manager threads", ie); } + + if (getConfig().getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) { + String hostname = getConfig().get(YarnConfiguration.RM_WEBAPP_ADDRESS, + YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS); + hostname = (hostname.contains(":")) ? hostname.substring(0, hostname.indexOf(":")) : hostname; + int port = webApp.port(); + String resolvedAddress = hostname + ":" + port; + conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS, resolvedAddress); + } super.start(); Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java?rev=1328109&r1=1328108&r2=1328109&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java Thu Apr 19 20:33:11 2012 @@ -133,6 +133,11 @@ public class ResourceTrackerService exte } this.server.start(); + if (getConfig().getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) { + String resolvedAddress = + server.getListenerAddress().getHostName() + ":" + server.getListenerAddress().getPort(); + conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, resolvedAddress); + } } @Override Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java?rev=1328109&r1=1328108&r2=1328109&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java Thu Apr 19 20:33:11 2012 @@ -20,6 +20,9 @@ package org.apache.hadoop.yarn.server; import java.io.File; import java.io.IOException; +import java.net.InetAddress; +import java.net.ServerSocket; +import java.net.UnknownHostException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -113,7 +116,16 @@ public class MiniYARNCluster extends Com public NodeManager getNodeManager(int i) { return this.nodeManagers[i]; } - + + public static String getHostname() { + try { + return InetAddress.getLocalHost().getHostName(); + } + catch (UnknownHostException ex) { + throw new RuntimeException(ex); + } + } + private class ResourceManagerWrapper extends AbstractService { public ResourceManagerWrapper() { super(ResourceManagerWrapper.class.getName()); @@ -122,6 +134,19 @@ public class MiniYARNCluster extends Com @Override public synchronized void start() { try { + getConfig().setBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, true); + getConfig().set(YarnConfiguration.RM_ADDRESS, + MiniYARNCluster.getHostname() + ":0"); + getConfig().set(YarnConfiguration.RM_ADDRESS, + MiniYARNCluster.getHostname() + ":0"); + getConfig().set(YarnConfiguration.RM_ADMIN_ADDRESS, + MiniYARNCluster.getHostname() + ":0"); + getConfig().set(YarnConfiguration.RM_SCHEDULER_ADDRESS, + MiniYARNCluster.getHostname() + ":0"); + getConfig().set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, + MiniYARNCluster.getHostname() + ":0"); + getConfig().set(YarnConfiguration.RM_WEBAPP_ADDRESS, + MiniYARNCluster.getHostname() + ":0"); Store store = StoreFactory.getStore(getConfig()); resourceManager = new ResourceManager(store) { @Override @@ -151,6 +176,10 @@ public class MiniYARNCluster extends Com } catch (Throwable t) { throw new YarnException(t); } + LOG.info("MiniYARN ResourceManager address: " + + getConfig().get(YarnConfiguration.RM_ADDRESS)); + LOG.info("MiniYARN ResourceManager web address: " + + getConfig().get(YarnConfiguration.RM_WEBAPP_ADDRESS)); } @Override @@ -212,9 +241,12 @@ public class MiniYARNCluster extends Com remoteLogDir.getAbsolutePath()); // By default AM + 2 containers getConfig().setInt(YarnConfiguration.NM_PMEM_MB, 4*1024); - getConfig().set(YarnConfiguration.NM_ADDRESS, "0.0.0.0:0"); - getConfig().set(YarnConfiguration.NM_LOCALIZER_ADDRESS, "0.0.0.0:0"); - getConfig().set(YarnConfiguration.NM_WEBAPP_ADDRESS, "0.0.0.0:0"); + getConfig().set(YarnConfiguration.NM_ADDRESS, + MiniYARNCluster.getHostname() + ":0"); + getConfig().set(YarnConfiguration.NM_LOCALIZER_ADDRESS, + MiniYARNCluster.getHostname() + ":0"); + getConfig().set(YarnConfiguration.NM_WEBAPP_ADDRESS, + MiniYARNCluster.getHostname() + ":0"); LOG.info("Starting NM: " + index); nodeManagers[index].init(getConfig()); new Thread() {