hadoop-yarn-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jl...@apache.org
Subject svn commit: r1409041 - in /hadoop/common/branches/branch-0.23/hadoop-yarn-project: ./ hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ hadoop-yarn/hadoop-...
Date Wed, 14 Nov 2012 00:06:05 GMT
Author: jlowe
Date: Wed Nov 14 00:06:01 2012
New Revision: 1409041

URL: http://svn.apache.org/viewvc?rev=1409041&view=rev
Log:
YARN-144. MiniMRYarnCluster launches RM and JHS on default ports. Contributed by Robert Parker

Added:
    hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/resources/yarn-site.xml
Modified:
    hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt
    hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
    hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
    hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
    hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
    hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java

Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt?rev=1409041&r1=1409040&r2=1409041&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt Wed Nov 14 00:06:01
2012
@@ -83,6 +83,9 @@ Release 0.23.5 - UNRELEASED
     YARN-212. NM state machine ignores an APPLICATION_CONTAINER_FINISHED event
     when it shouldn't (Nathan Roberts via jlowe)
 
+    YARN-144. MiniMRYarnCluster launches RM and JHS on default ports (Robert
+    Parker via jlowe)
+
 Release 0.23.4 - UNRELEASED
 
   INCOMPATIBLE CHANGES

Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java?rev=1409041&r1=1409040&r2=1409041&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
Wed Nov 14 00:06:01 2012
@@ -76,7 +76,6 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
 import org.apache.hadoop.yarn.ipc.YarnRPC;
-import org.apache.hadoop.yarn.security.client.ClientRMSecurityInfo;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.hadoop.yarn.util.Records;
 
@@ -198,13 +197,19 @@ public class Client {
 
   /**
    */
-  public Client() throws Exception  {
+  public Client(Configuration conf) throws Exception  {
     // Set up the configuration and RPC
-    conf = new Configuration();
+    this.conf = conf;
     rpc = YarnRPC.create(conf);
   }
 
   /**
+   */
+  public Client() throws Exception  {
+    this(new Configuration());
+  }
+
+  /**
    * Helper function to print out usage
    * @param opts Parsed command line options 
    */
@@ -504,22 +509,20 @@ public class Client {
     // It should be provided out of the box. 
     // For now setting all required classpaths including
     // the classpath to "." for the application jar
-    String classPathEnv = "${CLASSPATH}"
-        + ":./*"
-        + ":$HADOOP_CONF_DIR"
-        + ":$HADOOP_COMMON_HOME/share/hadoop/common/*"
-        + ":$HADOOP_COMMON_HOME/share/hadoop/common/lib/*"
-        + ":$HADOOP_HDFS_HOME/share/hadoop/hdfs/*"
-        + ":$HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/*"
-        + ":$YARN_HOME/modules/*"
-        + ":$YARN_HOME/lib/*"
-        + ":./log4j.properties:";
+    StringBuilder classPathEnv = new StringBuilder("${CLASSPATH}:./*");
+    for (String c : conf.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH)
+        .split(",")) {
+      classPathEnv.append(':');
+      classPathEnv.append(c.trim());
+    }
+    classPathEnv.append(":./log4j.properties");
 
-    // add the runtime classpath needed for tests to work 
+    // add the runtime classpath needed for tests to work
     String testRuntimeClassPath = Client.getTestRuntimeClasspath();
-    classPathEnv += ":" + testRuntimeClassPath; 
+    classPathEnv.append(':');
+    classPathEnv.append(testRuntimeClassPath);
 
-    env.put("CLASSPATH", classPathEnv);
+    env.put("CLASSPATH", classPathEnv.toString());
 
     amContainer.setEnvironment(env);
 

Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java?rev=1409041&r1=1409040&r2=1409041&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
Wed Nov 14 00:06:01 2012
@@ -18,11 +18,17 @@
 
 package org.apache.hadoop.yarn.applications.distributedshell;
 
+import java.io.File;
+import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.OutputStream;
+import java.net.URL;
 
+import junit.framework.Assert;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.util.JarFinder;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.MiniYARNCluster;
@@ -53,6 +59,14 @@ public class TestDistributedShell {
           1, 1, 1);
       yarnCluster.init(conf);
       yarnCluster.start();
+      URL url = Thread.currentThread().getContextClassLoader().getResource("yarn-site.xml");
+      if (url == null) {
+        throw new RuntimeException("Could not find 'yarn-site.xml' dummy file in classpath");
+      }
+      yarnCluster.getConfig().set("yarn.application.classpath", new File(url.getPath()).getParent());
+      OutputStream os = new FileOutputStream(new File(url.getPath()));
+      yarnCluster.getConfig().writeXml(os);
+      os.close();
     }
     try {
       Thread.sleep(2000);
@@ -86,14 +100,14 @@ public class TestDistributedShell {
     };
 
     LOG.info("Initializing DS Client");
-    Client client = new Client();
+    Client client = new Client(new Configuration(yarnCluster.getConfig()));
     boolean initSuccess = client.init(args);
-    assert(initSuccess);
+    Assert.assertTrue(initSuccess);
     LOG.info("Running DS Client");
     boolean result = client.run();
 
     LOG.info("Client run completed. Result=" + result);
-    assert (result == true);		 
+    Assert.assertTrue(result);
 
   }
 

Added: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/resources/yarn-site.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/resources/yarn-site.xml?rev=1409041&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/resources/yarn-site.xml
(added)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/resources/yarn-site.xml
Wed Nov 14 00:06:01 2012
@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+<configuration>
+  <!-- Dummy (invalid) config file to be overwriten by TestDistributedShell with MiniCluster
configuration. -->
+</configuration>
+
+

Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1409041&r1=1409040&r2=1409041&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
Wed Nov 14 00:06:01 2012
@@ -567,6 +567,18 @@ public class YarnConfiguration extends C
   /** Container temp directory */
   public static final String DEFAULT_CONTAINER_TEMP_DIR = "./tmp";
 
+  public static final String IS_MINI_YARN_CLUSTER = YARN_PREFIX + ".is.minicluster";
+
+  /** Whether to use fixed ports with the minicluster. */
+  public static final String YARN_MINICLUSTER_FIXED_PORTS = YARN_PREFIX
+      + "minicluster.fixed.ports";
+
+  /**
+   * Default is false to be able to run tests concurrently without port
+   * conflicts.
+   */
+  public static boolean DEFAULT_YARN_MINICLUSTER_FIXED_PORTS = false;
+
   public YarnConfiguration() {
     super();
   }

Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java?rev=1409041&r1=1409040&r2=1409041&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
Wed Nov 14 00:06:01 2012
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.re
 
 
 import java.io.IOException;
+import java.net.InetAddress;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingQueue;
 
@@ -506,6 +507,15 @@ public class ResourceManager extends Com
     } catch(IOException ie) {
       throw new YarnException("Failed to start secret manager threads", ie);
     }
+
+    if (getConfig().getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
+      String hostname = getConfig().get(YarnConfiguration.RM_WEBAPP_ADDRESS,
+                                        YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS);
+      hostname = (hostname.contains(":")) ? hostname.substring(0, hostname.indexOf(":"))
: hostname;
+      int port = webApp.port();
+      String resolvedAddress = hostname + ":" + port;
+      conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS, resolvedAddress);
+    }
     
     super.start();
 

Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java?rev=1409041&r1=1409040&r2=1409041&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
Wed Nov 14 00:06:01 2012
@@ -20,6 +20,9 @@ package org.apache.hadoop.yarn.server;
 
 import java.io.File;
 import java.io.IOException;
+import java.net.InetAddress;
+import java.net.ServerSocket;
+import java.net.UnknownHostException;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -118,7 +121,16 @@ public class MiniYARNCluster extends Com
   public NodeManager getNodeManager(int i) {
     return this.nodeManagers[i];
   }
-  
+
+  public static String getHostname() {
+    try {
+      return InetAddress.getLocalHost().getHostName();
+    }
+    catch (UnknownHostException ex) {
+      throw new RuntimeException(ex);
+    }
+  }
+
   private class ResourceManagerWrapper extends AbstractService {
     public ResourceManagerWrapper() {
       super(ResourceManagerWrapper.class.getName());
@@ -127,6 +139,22 @@ public class MiniYARNCluster extends Com
     @Override
     public synchronized void start() {
       try {
+        getConfig().setBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, true);
+        if (!getConfig().getBoolean(
+            YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS,
+            YarnConfiguration.DEFAULT_YARN_MINICLUSTER_FIXED_PORTS)) {
+          // pick free random ports.
+          getConfig().set(YarnConfiguration.RM_ADDRESS,
+              MiniYARNCluster.getHostname() + ":0");
+          getConfig().set(YarnConfiguration.RM_ADMIN_ADDRESS,
+              MiniYARNCluster.getHostname() + ":0");
+          getConfig().set(YarnConfiguration.RM_SCHEDULER_ADDRESS,
+              MiniYARNCluster.getHostname() + ":0");
+          getConfig().set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
+              MiniYARNCluster.getHostname() + ":0");
+          getConfig().set(YarnConfiguration.RM_WEBAPP_ADDRESS,
+              MiniYARNCluster.getHostname() + ":0");
+        }
         Store store = StoreFactory.getStore(getConfig());
         resourceManager = new ResourceManager(store) {
           @Override
@@ -156,6 +184,10 @@ public class MiniYARNCluster extends Com
       } catch (Throwable t) {
         throw new YarnException(t);
       }
+      LOG.info("MiniYARN ResourceManager address: " +
+               getConfig().get(YarnConfiguration.RM_ADDRESS));
+      LOG.info("MiniYARN ResourceManager web address: " +
+               getConfig().get(YarnConfiguration.RM_WEBAPP_ADDRESS));
     }
 
     @Override
@@ -217,9 +249,12 @@ public class MiniYARNCluster extends Com
             	remoteLogDir.getAbsolutePath());
         // By default AM + 2 containers
         getConfig().setInt(YarnConfiguration.NM_PMEM_MB, 4*1024);
-        getConfig().set(YarnConfiguration.NM_ADDRESS, "0.0.0.0:0");
-        getConfig().set(YarnConfiguration.NM_LOCALIZER_ADDRESS, "0.0.0.0:0");
-        getConfig().set(YarnConfiguration.NM_WEBAPP_ADDRESS, "0.0.0.0:0");
+        getConfig().set(YarnConfiguration.NM_ADDRESS,
+                        MiniYARNCluster.getHostname() + ":0");
+        getConfig().set(YarnConfiguration.NM_LOCALIZER_ADDRESS,
+                        MiniYARNCluster.getHostname() + ":0");
+        getConfig().set(YarnConfiguration.NM_WEBAPP_ADDRESS,
+                        MiniYARNCluster.getHostname() + ":0");
         LOG.info("Starting NM: " + index);
         nodeManagers[index].init(getConfig());
         new Thread() {



Mime
View raw message