hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jl...@apache.org
Subject hadoop git commit: YARN-4428. Redirect RM page to AHS page when AHS turned on and RM page is not available. Contributed by Chang Li (cherry picked from commit 772ea7b41b06beaa1f4ac4fa86eac8d6e6c8cd36)
Date Fri, 29 Jan 2016 21:55:21 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 885c76d85 -> 016c68513


YARN-4428. Redirect RM page to AHS page when AHS turned on and RM page is not available. Contributed
by Chang Li
(cherry picked from commit 772ea7b41b06beaa1f4ac4fa86eac8d6e6c8cd36)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/016c6851
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/016c6851
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/016c6851

Branch: refs/heads/branch-2
Commit: 016c6851351390643a8f281d02fd3f96cb3414e7
Parents: 885c76d
Author: Jason Lowe <jlowe@apache.org>
Authored: Fri Jan 29 21:48:54 2016 +0000
Committer: Jason Lowe <jlowe@apache.org>
Committed: Fri Jan 29 21:54:56 2016 +0000

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                 |  3 +
 .../rmapp/attempt/RMAppAttemptImpl.java         | 23 ++++-
 .../server/resourcemanager/webapp/RMWebApp.java |  5 ++
 .../resourcemanager/webapp/RMWebAppFilter.java  | 90 ++++++++++++++++++++
 .../attempt/TestRMAppAttemptTransitions.java    | 78 ++++++++++++++++-
 5 files changed, 197 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/016c6851/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 797331b..1efbcc3 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1386,6 +1386,9 @@ Release 2.7.3 - UNRELEASED
     YARN-4598. Invalid event: RESOURCE_FAILED at
     CONTAINER_CLEANEDUP_AFTER_KILL (tangshangwen via jlowe)
 
+    YARN-4428. Redirect RM page to AHS page when AHS turned on and RM page is
+    not available (Chang Li via jlowe)
+
 Release 2.7.2 - 2016-01-25
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/016c6851/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index 99f1b21..16456ae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -58,6 +58,7 @@ import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.factories.RecordFactory;
@@ -628,6 +629,21 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
     }
   }
 
+  private void setTrackingUrlToAHSPage(RMAppAttemptState stateToBeStored) {
+    originalTrackingUrl = pjoin(
+        WebAppUtils.getHttpSchemePrefix(conf) +
+        WebAppUtils.getAHSWebAppURLWithoutScheme(conf),
+        "applicationhistory", "app", getAppAttemptId().getApplicationId());
+    switch (stateToBeStored) {
+    case KILLED:
+    case FAILED:
+      proxiedTrackingUrl = originalTrackingUrl;
+      break;
+    default:
+      break;
+    }
+  }
+
   private void invalidateAMHostAndPort() {
     this.host = "N/A";
     this.rpcPort = -1;
@@ -1211,7 +1227,12 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable
{
     String diags = null;
 
     // don't leave the tracking URL pointing to a non-existent AM
-    setTrackingUrlToRMAppPage(stateToBeStored);
+    if (conf.getBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED,
+            YarnConfiguration.DEFAULT_APPLICATION_HISTORY_ENABLED)) {
+      setTrackingUrlToAHSPage(stateToBeStored);
+    } else {
+      setTrackingUrlToRMAppPage(stateToBeStored);
+    }
     String finalTrackingUrl = getOriginalTrackingUrl();
     FinalApplicationStatus finalStatus = null;
     int exitStatus = ContainerExitStatus.INVALID;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/016c6851/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
index 91c3d72..106065b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
 import org.apache.hadoop.yarn.api.ApplicationBaseProtocol;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.util.RMHAUtils;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
 import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
 import org.apache.hadoop.yarn.webapp.WebApp;
@@ -122,4 +123,8 @@ public class RMWebApp extends WebApp implements YarnWebParams {
     return rm.getRMContext().getRMAdminService()
       .getHAZookeeperConnectionState();
   }
+
+  public RMContext getRMContext() {
+    return rm.getRMContext();
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/016c6851/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java
index 9f35b6c..7413748 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java
@@ -18,6 +18,8 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.webapp;
 
+import static org.apache.hadoop.yarn.util.StringHelper.pjoin;
+
 import java.io.IOException;
 import java.io.PrintWriter;
 import java.net.InetSocketAddress;
@@ -35,9 +37,19 @@ import javax.servlet.http.HttpServletResponse;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.http.HtmlQuoting;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
 import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils;
+import org.apache.hadoop.yarn.util.Apps;
+import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.hadoop.yarn.webapp.YarnWebParams;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import com.google.common.collect.Sets;
 import com.google.inject.Injector;
@@ -45,6 +57,8 @@ import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
 
 @Singleton
 public class RMWebAppFilter extends GuiceContainer {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(RMWebAppFilter.class);
 
   private Injector injector;
   /**
@@ -56,6 +70,8 @@ public class RMWebAppFilter extends GuiceContainer {
   private static final Set<String> NON_REDIRECTED_URIS = Sets.newHashSet(
       "/conf", "/stacks", "/logLevel", "/logs");
   private String path;
+  private boolean ahsEnabled;
+  private String ahsPageURLPrefix;
   private static final int BASIC_SLEEP_TIME = 5;
   private static final int MAX_SLEEP_TIME = 5 * 60;
   private static final Random randnum = new Random();
@@ -76,6 +92,13 @@ public class RMWebAppFilter extends GuiceContainer {
     path = YarnConfiguration.useHttps(conf)
         ? "https://" + path
         : "http://" + path;
+    ahsEnabled = conf.getBoolean(
+        YarnConfiguration.APPLICATION_HISTORY_ENABLED,
+        YarnConfiguration.DEFAULT_APPLICATION_HISTORY_ENABLED);
+    ahsPageURLPrefix = pjoin(
+        WebAppUtils.getHttpSchemePrefix(conf) +
+        WebAppUtils.getAHSWebAppURLWithoutScheme(
+            conf), "applicationhistory");
   }
 
   @Override
@@ -137,11 +160,78 @@ public class RMWebAppFilter extends GuiceContainer {
         }
       }
       return;
+    } else if (ahsEnabled) {
+      String ahsRedirectUrl = ahsRedirectPath(uri, rmWebApp);
+      if(ahsRedirectUrl != null) {
+        response.setHeader("Location", ahsRedirectUrl);
+        response.setStatus(HttpServletResponse.SC_TEMPORARY_REDIRECT);
+        return;
+      }
     }
 
     super.doFilter(request, response, chain);
   }
 
+  private String ahsRedirectPath(String uri, RMWebApp rmWebApp) {
+    // TODO: Commonize URL parsing code. Will be done in YARN-4642.
+    String redirectPath = null;
+    if(uri.contains("/cluster/")) {
+      String[] parts = uri.split("/");
+      if(parts.length > 3) {
+        RMContext context = rmWebApp.getRMContext();
+        String type = parts[2];
+        ApplicationId appId = null;
+        ApplicationAttemptId appAttemptId = null;
+        ContainerId containerId = null;
+        switch(type){
+        case "app":
+          try {
+            appId = Apps.toAppID(parts[3]);
+          } catch (YarnRuntimeException | NumberFormatException e) {
+            LOG.debug("Error parsing {} as an ApplicationId",
+                parts[3], e);
+            return redirectPath;
+          }
+          if(!context.getRMApps().containsKey(appId)) {
+            redirectPath = pjoin(ahsPageURLPrefix, "app", appId);
+          }
+          break;
+        case "appattempt":
+          try{
+            appAttemptId = ConverterUtils.toApplicationAttemptId(parts[3]);
+          } catch (IllegalArgumentException e) {
+            LOG.debug("Error parsing {} as an ApplicationAttemptId",
+                parts[3], e);
+            return redirectPath;
+          }
+          if(!context.getRMApps().containsKey(
+              appAttemptId.getApplicationId())) {
+            redirectPath = pjoin(ahsPageURLPrefix,
+                "appattempt", appAttemptId);
+          }
+          break;
+        case "container":
+          try {
+            containerId = ContainerId.fromString(parts[3]);
+          } catch (IllegalArgumentException e) {
+            LOG.debug("Error parsing {} as an ContainerId",
+                parts[3], e);
+            return redirectPath;
+          }
+          if(!context.getRMApps().containsKey(
+              containerId.getApplicationAttemptId().getApplicationId())) {
+            redirectPath = pjoin(ahsPageURLPrefix,
+                "container", containerId);
+          }
+          break;
+        default:
+          break;
+        }
+      }
+    }
+    return redirectPath;
+  }
+
   private boolean shouldRedirect(RMWebApp rmWebApp, String uri) {
     return !uri.equals("/" + rmWebApp.wsName() + "/v1/cluster/info")
         && !uri.equals("/" + rmWebApp.name() + "/cluster")

http://git-wip-us.apache.org/repos/asf/hadoop/blob/016c6851/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
index e7985fa..93f17e7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
@@ -68,6 +68,7 @@ import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.AsyncDispatcher;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.event.InlineDispatcher;
@@ -137,7 +138,10 @@ public class TestRMAppAttemptTransitions {
   private static final String FAILED_DIAGNOSTICS = "Attempt failed by user.";
   private static final String RM_WEBAPP_ADDR =
       WebAppUtils.getResolvedRMWebAppURLWithScheme(new Configuration());
-  
+  private static final String AHS_WEBAPP_ADDR =
+      WebAppUtils.getHttpSchemePrefix(new Configuration()) +
+      WebAppUtils.getAHSWebAppURLWithoutScheme(new Configuration());
+
   private boolean isSecurityEnabled;
   private RMContext rmContext;
   private RMContext spyRMContext;
@@ -1090,6 +1094,78 @@ public class TestRMAppAttemptTransitions {
     verifyApplicationAttemptFinished(RMAppAttemptState.FAILED);
   }
 
+  @SuppressWarnings("unchecked")
+  @Test(timeout=10000)
+  public void testLaunchedFailWhileAHSEnabled() {
+    Configuration myConf = new Configuration(conf);
+    myConf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, true);
+    ApplicationId applicationId = MockApps.newAppID(appId);
+    ApplicationAttemptId applicationAttemptId =
+        ApplicationAttemptId.newInstance(applicationId, 2);
+    RMAppAttempt  myApplicationAttempt =
+        new RMAppAttemptImpl(applicationAttempt.getAppAttemptId(),
+            spyRMContext, scheduler,masterService,
+            submissionContext, myConf, false,
+            BuilderUtils.newResourceRequest(
+                RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY,
+                submissionContext.getResource(), 1));
+
+    //submit, schedule and allocate app attempt
+    myApplicationAttempt.handle(
+        new RMAppAttemptEvent(myApplicationAttempt.getAppAttemptId(),
+            RMAppAttemptEventType.START));
+    myApplicationAttempt.handle(
+        new RMAppAttemptEvent(
+            myApplicationAttempt.getAppAttemptId(),
+            RMAppAttemptEventType.ATTEMPT_ADDED));
+
+    Container amContainer = mock(Container.class);
+    Resource resource = BuilderUtils.newResource(2048, 1);
+    when(amContainer.getId()).thenReturn(
+        BuilderUtils.newContainerId(myApplicationAttempt.getAppAttemptId(), 1));
+    when(amContainer.getResource()).thenReturn(resource);
+    Allocation allocation = mock(Allocation.class);
+    when(allocation.getContainers()).
+        thenReturn(Collections.singletonList(amContainer));
+    when(scheduler.allocate(any(ApplicationAttemptId.class), any(List.class),
+        any(List.class), any(List.class), any(List.class), any(List.class),
+        any(List.class))).thenReturn(allocation);
+    RMContainer rmContainer = mock(RMContainerImpl.class);
+    when(scheduler.getRMContainer(amContainer.getId())).thenReturn(rmContainer);
+
+    myApplicationAttempt.handle(
+        new RMAppAttemptEvent(myApplicationAttempt.getAppAttemptId(),
+            RMAppAttemptEventType.CONTAINER_ALLOCATED));
+    assertEquals(RMAppAttemptState.ALLOCATED_SAVING,
+        myApplicationAttempt.getAppAttemptState());
+    myApplicationAttempt.handle(
+        new RMAppAttemptEvent(myApplicationAttempt.getAppAttemptId(),
+            RMAppAttemptEventType.ATTEMPT_NEW_SAVED));
+
+    // launch app attempt
+    myApplicationAttempt.handle(
+        new RMAppAttemptEvent(myApplicationAttempt.getAppAttemptId(),
+            RMAppAttemptEventType.LAUNCHED));
+    assertEquals(YarnApplicationAttemptState.LAUNCHED,
+        myApplicationAttempt.createApplicationAttemptState());
+
+    //fail container right after launched
+    NodeId anyNodeId = NodeId.newInstance("host", 1234);
+    myApplicationAttempt.handle(
+        new RMAppAttemptContainerFinishedEvent(
+            myApplicationAttempt.getAppAttemptId(),
+            BuilderUtils.newContainerStatus(amContainer.getId(),
+                ContainerState.COMPLETE, "", 0,
+                amContainer.getResource()), anyNodeId));
+    sendAttemptUpdateSavedEvent(myApplicationAttempt);
+    assertEquals(RMAppAttemptState.FAILED,
+        myApplicationAttempt.getAppAttemptState());
+    String rmAppPageUrl = pjoin(AHS_WEBAPP_ADDR, "applicationhistory", "app",
+        myApplicationAttempt.getAppAttemptId().getApplicationId());
+    assertEquals(rmAppPageUrl, myApplicationAttempt.getOriginalTrackingUrl());
+    assertEquals(rmAppPageUrl, myApplicationAttempt.getTrackingUrl());
+  }
+
   @Test(timeout=20000)
   public void testRunningExpire() {
     Container amContainer = allocateApplicationAttempt();


Mime
View raw message