hadoop-yarn-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject svn commit: r1532967 [9/9] - in /hadoop/common/branches/HDFS-4949/hadoop-yarn-project: ./ hadoop-yarn/bin/ hadoop-yarn/conf/ hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ hadoop-yarn/hadoop-yarn-api/src/main/java...
Date Thu, 17 Oct 2013 05:33:06 GMT
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java?rev=1532967&r1=1532966&r2=1532967&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
(original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
Thu Oct 17 05:32:42 2013
@@ -28,6 +28,7 @@ import org.apache.commons.logging.LogFac
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileContext;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.http.HttpConfig;
 import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
 import org.apache.hadoop.service.AbstractService;
 import org.apache.hadoop.service.CompositeService;
@@ -39,7 +40,6 @@ import org.apache.hadoop.yarn.exceptions
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
-import org.apache.hadoop.yarn.ipc.RPCUtil;
 import org.apache.hadoop.yarn.server.api.ResourceTracker;
 import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest;
 import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
@@ -52,6 +52,7 @@ import org.apache.hadoop.yarn.server.nod
 import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceTrackerService;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
 
 /**
  * Embedded Yarn minicluster for testcases that need to interact with a cluster.
@@ -188,32 +189,33 @@ public class MiniYARNCluster extends Com
     }
 
     @Override
-    public synchronized void serviceStart() throws Exception {
-      try {
-        getConfig().setBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, true);
-        if (!getConfig().getBoolean(
-            YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS,
-            YarnConfiguration.DEFAULT_YARN_MINICLUSTER_FIXED_PORTS)) {
-          // pick free random ports.
-          String hostname = MiniYARNCluster.getHostname();
-          getConfig().set(YarnConfiguration.RM_ADDRESS,
-              hostname + ":0");
-          getConfig().set(YarnConfiguration.RM_ADMIN_ADDRESS,
-              hostname + ":0");
-          getConfig().set(YarnConfiguration.RM_SCHEDULER_ADDRESS,
-              hostname + ":0");
-          getConfig().set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
-              hostname + ":0");
-          getConfig().set(YarnConfiguration.RM_WEBAPP_ADDRESS,
-              hostname + ":0");
-        }
-        resourceManager = new ResourceManager() {
-          @Override
-          protected void doSecureLogin() throws IOException {
-            // Don't try to login using keytab in the testcase.
-          };
+    protected synchronized void serviceInit(Configuration conf)
+        throws Exception {
+      conf.setBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, true);
+      if (!conf.getBoolean(
+          YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS,
+          YarnConfiguration.DEFAULT_YARN_MINICLUSTER_FIXED_PORTS)) {
+        // pick free random ports.
+        String hostname = MiniYARNCluster.getHostname();
+        conf.set(YarnConfiguration.RM_ADDRESS, hostname + ":0");
+        conf.set(YarnConfiguration.RM_ADMIN_ADDRESS, hostname + ":0");
+        conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, hostname + ":0");
+        conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, hostname + ":0");
+        WebAppUtils.setRMWebAppHostnameAndPort(conf, hostname, 0);
+      }
+      resourceManager = new ResourceManager() {
+        @Override
+        protected void doSecureLogin() throws IOException {
+          // Don't try to login using keytab in the testcase.
         };
-        resourceManager.init(getConfig());
+      };
+      resourceManager.init(conf);
+      super.serviceInit(conf);
+    }
+
+    @Override
+    protected synchronized void serviceStart() throws Exception {
+      try {
         new Thread() {
           public void run() {
             resourceManager.start();
@@ -238,11 +240,11 @@ public class MiniYARNCluster extends Com
       LOG.info("MiniYARN ResourceManager address: " +
                getConfig().get(YarnConfiguration.RM_ADDRESS));
       LOG.info("MiniYARN ResourceManager web address: " +
-               getConfig().get(YarnConfiguration.RM_WEBAPP_ADDRESS));
+               WebAppUtils.getRMWebAppURLWithoutScheme(getConfig()));
     }
 
     @Override
-    public synchronized void serviceStop() throws Exception {
+    protected synchronized void serviceStop() throws Exception {
       if (resourceManager != null) {
         resourceManager.stop();
       }
@@ -271,8 +273,43 @@ public class MiniYARNCluster extends Com
       index = i;
     }
 
-    public synchronized void serviceInit(Configuration conf) throws Exception {
+    protected synchronized void serviceInit(Configuration conf)
+        throws Exception {
       Configuration config = new YarnConfiguration(conf);
+      // create nm-local-dirs and configure them for the nodemanager
+      String localDirsString = prepareDirs("local", numLocalDirs);
+      config.set(YarnConfiguration.NM_LOCAL_DIRS, localDirsString);
+      // create nm-log-dirs and configure them for the nodemanager
+      String logDirsString = prepareDirs("log", numLogDirs);
+      config.set(YarnConfiguration.NM_LOG_DIRS, logDirsString);
+
+      File remoteLogDir =
+          new File(testWorkDir, MiniYARNCluster.this.getName()
+              + "-remoteLogDir-nm-" + index);
+      remoteLogDir.mkdir();
+      config.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
+          remoteLogDir.getAbsolutePath());
+      // By default AM + 2 containers
+      config.setInt(YarnConfiguration.NM_PMEM_MB, 4*1024);
+      config.set(YarnConfiguration.NM_ADDRESS,
+          MiniYARNCluster.getHostname() + ":0");
+      config.set(YarnConfiguration.NM_LOCALIZER_ADDRESS,
+          MiniYARNCluster.getHostname() + ":0");
+      WebAppUtils
+          .setNMWebAppHostNameAndPort(config,
+              MiniYARNCluster.getHostname(), 0);
+
+      // Disable resource checks by default
+      if (!config.getBoolean(
+          YarnConfiguration.YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING,
+          YarnConfiguration.
+              DEFAULT_YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING)) {
+        config.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, false);
+        config.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, false);
+      }
+
+      LOG.info("Starting NM: " + index);
+      nodeManagers[index].init(config);
       super.serviceInit(config);
     }
 
@@ -296,41 +333,8 @@ public class MiniYARNCluster extends Com
       return dirsString;
     }
 
-    public synchronized void serviceStart() throws Exception {
+    protected synchronized void serviceStart() throws Exception {
       try {
-        // create nm-local-dirs and configure them for the nodemanager
-        String localDirsString = prepareDirs("local", numLocalDirs);
-        getConfig().set(YarnConfiguration.NM_LOCAL_DIRS, localDirsString);
-        // create nm-log-dirs and configure them for the nodemanager
-        String logDirsString = prepareDirs("log", numLogDirs);
-        getConfig().set(YarnConfiguration.NM_LOG_DIRS, logDirsString);
-
-        File remoteLogDir =
-            new File(testWorkDir, MiniYARNCluster.this.getName()
-                + "-remoteLogDir-nm-" + index);
-        remoteLogDir.mkdir();
-        getConfig().set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
-            	remoteLogDir.getAbsolutePath());
-        // By default AM + 2 containers
-        getConfig().setInt(YarnConfiguration.NM_PMEM_MB, 4*1024);
-        getConfig().set(YarnConfiguration.NM_ADDRESS,
-                        MiniYARNCluster.getHostname() + ":0");
-        getConfig().set(YarnConfiguration.NM_LOCALIZER_ADDRESS,
-                        MiniYARNCluster.getHostname() + ":0");
-        getConfig().set(YarnConfiguration.NM_WEBAPP_ADDRESS,
-                        MiniYARNCluster.getHostname() + ":0");
-
-        // Disable resource checks by default
-        if (!getConfig().getBoolean(
-            YarnConfiguration.YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING,
-            YarnConfiguration.
-                DEFAULT_YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING)) {
-          getConfig().setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, false);
-          getConfig().setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, false);
-        }
-
-        LOG.info("Starting NM: " + index);
-        nodeManagers[index].init(getConfig());
         new Thread() {
           public void run() {
             nodeManagers[index].start();
@@ -353,7 +357,7 @@ public class MiniYARNCluster extends Com
     }
 
     @Override
-    public synchronized void serviceStop() throws Exception {
+    protected synchronized void serviceStop() throws Exception {
       if (nodeManagers[index] != null) {
         nodeManagers[index].stop();
       }

Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java?rev=1532967&r1=1532966&r2=1532967&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java
(original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java
Thu Oct 17 05:32:42 2013
@@ -18,7 +18,11 @@
 
 package org.apache.hadoop.yarn.server.webproxy;
 
-import static org.apache.hadoop.yarn.util.StringHelper.ujoin;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.http.HttpConfig;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.util.TrackingUriPlugin;
 
 import java.io.UnsupportedEncodingException;
 import java.net.URI;
@@ -26,11 +30,7 @@ import java.net.URISyntaxException;
 import java.net.URLEncoder;
 import java.util.List;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.http.HttpConfig;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.util.TrackingUriPlugin;
+import static org.apache.hadoop.yarn.util.StringHelper.ujoin;
 
 public class ProxyUriUtils {
   @SuppressWarnings("unused")
@@ -137,14 +137,44 @@ public class ProxyUriUtils {
   
   /**
    * Create a URI form a no scheme Url, such as is returned by the AM.
-   * @param noSchemeUrl the URL formate returned by an AM
+   * @param url the URL format returned by an AM. This may or may not contain
+   * scheme.
    * @return a URI with an http scheme
    * @throws URISyntaxException if the url is not formatted correctly.
    */
-  public static URI getUriFromAMUrl(String noSchemeUrl)
+  public static URI getUriFromAMUrl(String url)
     throws URISyntaxException {
-      return new URI(HttpConfig.getSchemePrefix() + noSchemeUrl);
+    if (getSchemeFromUrl(url).isEmpty()) {
+      /*
+       * check is made to make sure if AM reports with scheme then it will be
+       * used by default otherwise it will default to the one configured using
+       * "yarn.http.policy".
+       */
+      return new URI(HttpConfig.getSchemePrefix() + url);
+    } else {
+      return new URI(url);
+    }
   }
+  
+  /**
+   * Create a URI form a no scheme Url, such as is returned by the AM.
+   * @param noSchemeUrl the URL formate returned by an AM
+   * @return a URI with an http scheme
+   * @throws URISyntaxException if the url is not formatted correctly.
+   */
+  public static URI getUriFromAMUrl(String scheme, String noSchemeUrl)
+      throws URISyntaxException {
+      if (getSchemeFromUrl(noSchemeUrl).isEmpty()) {
+        /*
+         * check is made to make sure if AM reports with scheme then it will be
+         * used by default otherwise it will default to the one configured using
+         * "yarn.http.policy".
+         */
+        return new URI(scheme + "://" + noSchemeUrl);
+      } else {
+        return new URI(noSchemeUrl);
+      }
+    }
 
   /**
    * Returns the first valid tracking link, if any, from the given id from the
@@ -169,4 +199,20 @@ public class ProxyUriUtils {
     }
     return null;
   }
+  
+  /**
+   * Returns the scheme if present in the url
+   * eg. "https://issues.apache.org/jira/browse/YARN" > "https"
+   */
+  public static String getSchemeFromUrl(String url) {
+    int index = 0;
+    if (url != null) {
+      index = url.indexOf("://");
+    }
+    if (index > 0) {
+      return url.substring(0, index);
+    } else {
+      return "";
+    }
+  }
 }

Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java?rev=1532967&r1=1532966&r2=1532967&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java
(original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java
Thu Oct 17 05:32:42 2013
@@ -29,6 +29,7 @@ import org.apache.hadoop.service.Abstrac
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
 
 import org.apache.hadoop.fs.CommonConfigurationKeys;
 
@@ -62,7 +63,7 @@ public class WebAppProxy extends Abstrac
           CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION +
           " of " + auth);
     }
-    String proxy = YarnConfiguration.getProxyHostAndPort(conf);
+    String proxy = WebAppUtils.getProxyHostAndPort(conf);
     String[] proxyParts = proxy.split(":");
     proxyHost = proxyParts[0];
 

Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServer.java?rev=1532967&r1=1532966&r2=1532967&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServer.java
(original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServer.java
Thu Oct 17 05:32:42 2013
@@ -73,30 +73,32 @@ public class WebAppProxyServer extends C
         YarnConfiguration.PROXY_PRINCIPAL);
   }
 
-  /**
-   * Wait for service to finish.
-   * (Normally, it runs forever.)
-   */
-  private void join() {
-    proxy.join();
-  }
-
   public static void main(String[] args) {
     Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
     StringUtils.startupShutdownMessage(WebAppProxyServer.class, args, LOG);
     try {
-      WebAppProxyServer proxy = new WebAppProxyServer();
-      ShutdownHookManager.get().addShutdownHook(
-        new CompositeServiceShutdownHook(proxy),
-        SHUTDOWN_HOOK_PRIORITY);
-      YarnConfiguration conf = new YarnConfiguration();
-      proxy.init(conf);
-      proxy.start();
-      proxy.join();
+      WebAppProxyServer proxyServer = startServer();
+      proxyServer.proxy.join();
     } catch (Throwable t) {
       LOG.fatal("Error starting Proxy server", t);
       System.exit(-1);
     }
   }
 
+  /**
+   * Start proxy server.
+   * 
+   * @return proxy server instance.
+   */
+  protected static WebAppProxyServer startServer() throws Exception {
+    WebAppProxyServer proxy = new WebAppProxyServer();
+    ShutdownHookManager.get().addShutdownHook(
+        new CompositeServiceShutdownHook(proxy), SHUTDOWN_HOOK_PRIORITY);
+    YarnConfiguration configuration = new YarnConfiguration();
+    configuration.set(YarnConfiguration.PROXY_ADDRESS, "localhost:9099");
+    proxy.init(configuration);
+    proxy.start();
+    return proxy;
+  }
+
 }

Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java?rev=1532967&r1=1532966&r2=1532967&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
(original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
Thu Oct 17 05:32:42 2013
@@ -56,6 +56,7 @@ import org.apache.hadoop.yarn.util.Strin
 import org.apache.hadoop.yarn.util.TrackingUriPlugin;
 import org.apache.hadoop.yarn.webapp.MimeType;
 import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
 
 public class WebAppProxyServlet extends HttpServlet {
   private static final long serialVersionUID = 1L;
@@ -94,7 +95,7 @@ public class WebAppProxyServlet extends 
         conf.getInstances(YarnConfiguration.YARN_TRACKING_URL_GENERATOR,
             TrackingUriPlugin.class);
     this.rmAppPageUrlBase = StringHelper.pjoin(
-        YarnConfiguration.getRMWebAppURL(conf), "cluster", "app");
+        WebAppUtils.getResolvedRMWebAppURLWithScheme(conf), "cluster", "app");
   }
 
   /**
@@ -163,7 +164,6 @@ public class WebAppProxyServlet extends 
     }
     config.setLocalAddress(localAddress);
     HttpMethod method = new GetMethod(uri.getEscapedURI());
-
     @SuppressWarnings("unchecked")
     Enumeration<String> names = req.getHeaderNames();
     while(names.hasMoreElements()) {
@@ -293,14 +293,17 @@ public class WebAppProxyServlet extends 
       }
       String original = applicationReport.getOriginalTrackingUrl();
       URI trackingUri = null;
-      if (original != null) {
-        trackingUri = ProxyUriUtils.getUriFromAMUrl(original);
-      }
       // fallback to ResourceManager's app page if no tracking URI provided
       if(original == null || original.equals("N/A")) {
         resp.sendRedirect(resp.encodeRedirectURL(
             StringHelper.pjoin(rmAppPageUrlBase, id.toString())));
         return;
+      } else {
+        if (ProxyUriUtils.getSchemeFromUrl(original).isEmpty()) {
+          trackingUri = ProxyUriUtils.getUriFromAMUrl("http", original);
+        } else {
+          trackingUri = new URI(original);
+        }
       }
 
       String runningUser = applicationReport.getUser();
@@ -311,8 +314,7 @@ public class WebAppProxyServlet extends 
             req.getQueryString(), true), runningUser, id);
         return;
       }
-      
-      URI toFetch = new URI(req.getScheme(), 
+      URI toFetch = new URI(trackingUri.getScheme(), 
           trackingUri.getAuthority(),
           StringHelper.ujoin(trackingUri.getPath(), rest), req.getQueryString(),
           null);

Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java?rev=1532967&r1=1532966&r2=1532967&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java
(original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java
Thu Oct 17 05:32:42 2013
@@ -26,7 +26,7 @@ import org.apache.hadoop.http.FilterCont
 import org.apache.hadoop.http.FilterInitializer;
 import org.apache.hadoop.http.HttpConfig;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
 
 public class AmFilterInitializer extends FilterInitializer {
   private static final String FILTER_NAME = "AM_PROXY_FILTER";
@@ -35,7 +35,7 @@ public class AmFilterInitializer extends
   @Override
   public void initFilter(FilterContainer container, Configuration conf) {
     Map<String, String> params = new HashMap<String, String>();
-    String proxy = YarnConfiguration.getProxyHostAndPort(conf);
+    String proxy = WebAppUtils.getProxyHostAndPort(conf);
     String[] parts = proxy.split(":");
     params.put(AmIpFilter.PROXY_HOST, parts[0]);
     params.put(AmIpFilter.PROXY_URI_BASE,

Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm?rev=1532967&r1=1532966&r2=1532967&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
(original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
Thu Oct 17 05:32:42 2013
@@ -81,7 +81,7 @@ Hadoop MapReduce Next Generation - Fair 
   distribute the resources assigned to them to their children in the same
   fashion.  Applications may only be scheduled on leaf queues. Queues can be
   specified as children of other queues by placing them as sub-elements of 
-  their parents in the fair scheduler configuration file.
+  their parents in the fair scheduler allocation file.
   
   A queue's name starts with the names of its parents, with periods as
   separators. So a queue named "queue1" under the root queue, would be referred
@@ -118,16 +118,20 @@ Hadoop MapReduce Next Generation - Fair 
   Customizing the Fair Scheduler typically involves altering two files. First, 
   scheduler-wide options can be set by adding configuration properties in the 
   yarn-site.xml file in your existing configuration directory. Second, in 
-  most cases users will want to create a manifest file listing which queues 
-  exist and their respective weights and capacities. The location of this file 
-  is flexible - but it must be declared in yarn-site.xml.
+  most cases users will want to create an allocation file listing which queues 
+  exist and their respective weights and capacities. The allocation file
+  is reloaded every 10 seconds, allowing changes to be made on the fly.
+
+Properties that can be placed in yarn-site.xml
 
  * <<<yarn.scheduler.fair.allocation.file>>>
 
    * Path to allocation file. An allocation file is an XML manifest describing
      queues and their properties, in addition to certain policy defaults. This file
-     must be in XML format as described in the next section.
-     Defaults to fair-scheduler.xml in configuration directory.
+     must be in the XML format described in the next section. If a relative path is
+     given, the file is searched for on the classpath (which typically includes
+     the Hadoop conf directory).
+     Defaults to fair-scheduler.xml.
 
  * <<<yarn.scheduler.fair.user-as-default-queue>>>
 

Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm?rev=1532967&r1=1532966&r2=1532967&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
(original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
Thu Oct 17 05:32:42 2013
@@ -1269,6 +1269,129 @@ _01_000001</amContainerLogs>
 
 +---+
 
+* Cluster Application Statistics API
+
+  With the Application Statistics API, you can obtain a collection of triples, each of which
contains the application type, the application state and the number of applications of this
type and this state in ResourceManager context. Note that with the performance concern, we
currently only support at most one applicationType per query. We may support multiple applicationTypes
per query as well as more statistics in the future. When you run a GET operation on this resource,
you obtain a collection of statItem objects. 
+
+** URI
+
+------
+  * http://<rm http address:port>/ws/v1/cluster/appstatistics
+------
+
+** HTTP Operations Supported
+
+------
+  * GET
+------
+
+** Query Parameters Required
+
+  Two paramters can be specified. The parameters are case insensitive.
+
+------
+  * states - states of the applications, specified as a comma-separated list. If states is
not provided, the API will enumerate all application states and return the counts of them.
+  * applicationTypes - types of the applications, specified as a comma-separated list. If
applicationTypes is not provided, the API will count the applications of any application type.
In this case, the response shows * to indicate any application type. Note that we only support
at most one applicationType temporarily. Otherwise, users will expect an BadRequestException.
+------
+
+** Elements of the <appStatInfo> (statItems) object
+
+  When you make a request for the list of statistics items, the information will be returned
as a collection of statItem objects
+
+*-----------+----------------------------------------------------------------------+-------------------------------------+
+|| Item     || Data Type                                                           || Description
                       |
+*-----------+----------------------------------------------------------------------+-------------------------------------+
+| statItem  | array of statItem objects(JSON)/zero or more statItem objects(XML)   | The
collection of statItem objects  |
+*-----------+----------------------------------------------------------------------+-------------------------------------+
+
+** Response Examples
+
+  <<JSON response>>
+
+  HTTP Request:
+
+------
+  GET http://<rm http address:port>/ws/v1/cluster/appstatistics?states=accepted,running,finished&applicationTypes=mapreduce
+------
+
+  Response Header:
+
++---+
+  HTTP/1.1 200 OK
+  Content-Type: application/json
+  Transfer-Encoding: chunked
+  Server: Jetty(6.1.26)
++---+
+
+  Response Body:
+
++---+
+{
+  "appStatInfo":
+  {
+    "statItem":
+    [
+       {
+          "state" : "accepted",
+          "type" : "mapreduce",
+          "count" : 4
+       },
+       {
+          "state" : "running",
+          "type" : "mapreduce",
+          "count" : 1
+       },
+       {
+          "state" : "finished",
+          "type" : "mapreduce",
+          "count" : 7
+       }
+    ]
+  }
+}
++---+
+
+  <<XML response>>
+
+  HTTP Request:
+
+------
+  GET http://<rm http address:port>/ws/v1/cluster/appstatistics?states=accepted,running,finished&applicationTypes=mapreduce
+  Accept: application/xml
+------
+
+  Response Header:
+
++---+
+  HTTP/1.1 200 OK
+  Content-Type: application/xml
+  Content-Length: 2459
+  Server: Jetty(6.1.26)
++---+
+
+  Response Body:
+
++---+
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<appStatInfo>
+  <statItem>
+    <state>accepted</state>
+    <type>mapreduce</type>
+    <count>4</count>
+  </statItem>
+  <statItem>
+    <state>running</state>
+    <type>mapreduce</type>
+    <count>1</count>
+  </statItem>
+  <statItem>
+    <state>finished</state>
+    <type>mapreduce</type>
+    <count>7</count>
+  </statItem>
+</appStatInfo>
++---+
+
 * Cluster {Application API}
 
   An application resource contains information about a particular application that was submitted
to a cluster.



Mime
View raw message