hadoop-yarn-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vino...@apache.org
Subject svn commit: r1425214 - in /hadoop/common/branches/branch-0.23/hadoop-yarn-project: ./ hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/ hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ hadoop-yarn/hadoop...
Date Sat, 22 Dec 2012 00:33:14 GMT
Author: vinodkv
Date: Sat Dec 22 00:33:14 2012
New Revision: 1425214

URL: http://svn.apache.org/viewvc?rev=1425214&view=rev
Log:
YARN-285. Added a temporary plugin interface for RM to be able to redirect to JobHistory server
for apps that it no longer tracks. Contributed by Derek Dagit.
svn merge --ignore-ancestry -c 1425210 ../../trunk/

Added:
    hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/TrackingUriPlugin.java
Modified:
    hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt
    hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
    hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java
    hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
    hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyUriUtils.java

Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt?rev=1425214&r1=1425213&r2=1425214&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt Sat Dec 22 00:33:14
2012
@@ -8,6 +8,10 @@ Release 0.23.6 - UNRELEASED
 
   IMPROVEMENTS
 
+    YARN-285. Added a temporary plugin interface for RM to be able to redirect to
+    JobHistory server for apps that it no longer tracks. Contributed by
+    Derek Dagit.
+
   OPTIMIZATIONS
 
   BUG FIXES

Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1425214&r1=1425213&r2=1425214&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
Sat Dec 22 00:33:14 2012
@@ -399,6 +399,9 @@ public class YarnConfiguration extends C
 
   public static final String YARN_LOG_SERVER_URL =
     YARN_PREFIX + "log.server.url";
+  
+  public static final String YARN_TRACKING_URL_GENERATOR = 
+      YARN_PREFIX + "tracking.url.generator";
 
   /** Amount of memory in GB that can be allocated for containers.*/
   public static final String NM_PMEM_MB = NM_PREFIX + "resource.memory-mb";

Added: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/TrackingUriPlugin.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/TrackingUriPlugin.java?rev=1425214&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/TrackingUriPlugin.java
(added)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/TrackingUriPlugin.java
Sat Dec 22 00:33:14 2012
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.util;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+
+/**
+ * Plugin to derive a tracking URL from a Yarn Application ID
+ *
+ */
+@InterfaceAudience.LimitedPrivate({"MapReduce"})
+@InterfaceStability.Unstable
+public abstract class TrackingUriPlugin extends Configured {
+
+  /**
+   * Given an application ID, return a tracking URI.
+   * @param id the ID for which a URI is returned
+   * @return the tracking URI
+   * @throws URISyntaxException
+   */
+  public abstract URI getTrackingUri(ApplicationId id)
+      throws URISyntaxException;
+
+}

Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java?rev=1425214&r1=1425213&r2=1425214&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java
Sat Dec 22 00:33:14 2012
@@ -24,10 +24,12 @@ import java.io.UnsupportedEncodingExcept
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URLEncoder;
+import java.util.List;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.util.TrackingUriPlugin;
 
 public class ProxyUriUtils {
   @SuppressWarnings("unused")
@@ -138,8 +140,32 @@ public class ProxyUriUtils {
    * @return a URI with an http scheme
    * @throws URISyntaxException if the url is not formatted correctly.
    */
-  public static URI getUriFromAMUrl(String noSchemeUrl) 
+  public static URI getUriFromAMUrl(String noSchemeUrl)
     throws URISyntaxException {
     return new URI("http://"+noSchemeUrl);
   }
+
+  /**
+   * Returns the first valid tracking link, if any, from the given id from the
+   * given list of plug-ins, if any.
+   * 
+   * @param id the id of the application for which the tracking link is desired
+   * @param trackingUriPlugins list of plugins from which to get the tracking link
+   * @return the desired link if possible, otherwise null
+   * @throws URISyntaxException
+   */
+  public static URI getUriFromTrackingPlugins(ApplicationId id,
+      List<TrackingUriPlugin> trackingUriPlugins)
+      throws URISyntaxException {
+    URI toRet = null;
+    for(TrackingUriPlugin plugin : trackingUriPlugins)
+    {
+      toRet = plugin.getTrackingUri(id);
+      if (toRet != null)
+      {
+        return toRet;
+      }
+    }
+    return null;
+  }
 }

Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java?rev=1425214&r1=1425213&r2=1425214&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
Sat Dec 22 00:33:14 2012
@@ -30,6 +30,7 @@ import java.util.Arrays;
 import java.util.EnumSet;
 import java.util.Enumeration;
 import java.util.HashSet;
+import java.util.List;
 
 import javax.servlet.http.Cookie;
 import javax.servlet.http.HttpServlet;
@@ -48,8 +49,10 @@ import org.apache.commons.logging.LogFac
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.util.Apps;
 import org.apache.hadoop.yarn.util.StringHelper;
+import org.apache.hadoop.yarn.util.TrackingUriPlugin;
 import org.apache.hadoop.yarn.webapp.MimeType;
 import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
 
@@ -61,8 +64,9 @@ public class WebAppProxyServlet extends 
         "Accept-Language", "Accept-Charset"));
   
   public static final String PROXY_USER_COOKIE_NAME = "proxy-user";
-  
-  
+
+  private final List<TrackingUriPlugin> trackingUriPlugins;
+
   private static class _ implements Hamlet._ {
     //Empty
   }
@@ -78,6 +82,18 @@ public class WebAppProxyServlet extends 
   }
 
   /**
+   * Default constructor
+   */
+  public WebAppProxyServlet()
+  {
+    super();
+    YarnConfiguration conf = new YarnConfiguration();
+    this.trackingUriPlugins =
+        conf.getInstances(YarnConfiguration.YARN_TRACKING_URL_GENERATOR,
+            TrackingUriPlugin.class);
+  }
+
+  /**
    * Output 404 with appropriate message.
    * @param resp the http response.
    * @param message the message to include on the page.
@@ -253,6 +269,16 @@ public class WebAppProxyServlet extends 
       if(applicationReport == null) {
         LOG.warn(req.getRemoteUser()+" Attempting to access "+id+
             " that was not found");
+
+        URI toFetch =
+            ProxyUriUtils
+                .getUriFromTrackingPlugins(id, this.trackingUriPlugins);
+        if (toFetch != null)
+        {
+          resp.sendRedirect(resp.encodeRedirectURL(toFetch.toString()));
+          return;
+        }
+
         notFound(resp, "Application "+appId+" could not be found, " +
         		"please try the history server");
         return;

Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyUriUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyUriUtils.java?rev=1425214&r1=1425213&r2=1425214&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyUriUtils.java
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyUriUtils.java
Sat Dec 22 00:33:14 2012
@@ -21,11 +21,16 @@ package org.apache.hadoop.yarn.server.we
 import static org.junit.Assert.*;
 
 import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.List;
 
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.util.BuilderUtils;
+import org.apache.hadoop.yarn.util.TrackingUriPlugin;
 import org.junit.Test;
 
+import com.google.common.collect.Lists;
+
 public class TestProxyUriUtils {
   @Test
   public void testGetPathApplicationId() {
@@ -83,4 +88,36 @@ public class TestProxyUriUtils {
     URI result = ProxyUriUtils.getProxyUri(originalUri, proxyUri, id);
     assertEquals(expected, result);
   }
+
+  @Test
+  public void testGetProxyUriFromPluginsReturnsNullIfNoPlugins()
+      throws URISyntaxException {
+    ApplicationId id = BuilderUtils.newApplicationId(6384623l, 5);
+    List<TrackingUriPlugin> list =
+        Lists.newArrayListWithExpectedSize(0);
+    assertNull(ProxyUriUtils.getUriFromTrackingPlugins(id, list));
+  }
+
+  @Test
+  public void testGetProxyUriFromPluginsReturnsValidUriWhenAble()
+      throws URISyntaxException {
+    ApplicationId id = BuilderUtils.newApplicationId(6384623l, 5);
+    List<TrackingUriPlugin> list =
+        Lists.newArrayListWithExpectedSize(2);
+    // Insert a plugin that returns null.
+    list.add(new TrackingUriPlugin() {
+      public URI getTrackingUri(ApplicationId id) throws URISyntaxException {
+        return null;
+      }
+    });
+    // Insert a plugin that returns a valid URI.
+    list.add(new TrackingUriPlugin() {
+      public URI getTrackingUri(ApplicationId id) throws URISyntaxException {
+        return new URI("http://history.server.net/");
+      }
+    });
+    URI result = ProxyUriUtils.getUriFromTrackingPlugins(id, list);
+    assertNotNull(result);
+    
+  }
 }



Mime
View raw message