hadoop-yarn-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zjs...@apache.org
Subject svn commit: r1556752 - in /hadoop/common/branches/YARN-321/hadoop-yarn-project: ./ hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/ hadoop-yarn/ha...
Date Thu, 09 Jan 2014 07:24:23 GMT
Author: zjshen
Date: Thu Jan  9 07:24:22 2014
New Revision: 1556752

URL: http://svn.apache.org/r1556752
Log:
YARN-1413. Implemented serving of aggregated-logs in the ApplicationHistory server. Contributed
by Mayank Bansal.
svn merge --ignore-ancestry -c 1555005 ../../branches/YARN-321-old/

Added:
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSLogsPage.java
      - copied unchanged from r1555005, hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSLogsPage.java
Modified:
    hadoop/common/branches/YARN-321/hadoop-yarn-project/CHANGES.txt
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSController.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java
    hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java

Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/CHANGES.txt?rev=1556752&r1=1556751&r2=1556752&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/CHANGES.txt Thu Jan  9 07:24:22 2014
@@ -464,6 +464,9 @@ Branch YARN-321: Generic ApplicationHist
 
   YARN-1534. Fixed failure of test TestAHSWebApp. (Shinichi Yamashita via vinodkv)
 
+  YARN-1413. Implemented serving of aggregated-logs in the ApplicationHistory
+  server. (Mayank Bansal via vinodkv)
+
 Release 2.2.0 - 2013-10-13
 
   INCOMPATIBLE CHANGES

Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java?rev=1556752&r1=1556751&r2=1556752&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java
(original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java
Thu Jan  9 07:24:22 2014
@@ -178,4 +178,8 @@ public final class StringHelper {
   public static String percent(double value) {
     return String.format("%.2f", value * 100);
   }
+  
+  public static String getPartUrl(String url, String part) {
+    return url.substring(url.indexOf(part));
+  }
 }

Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java?rev=1556752&r1=1556751&r2=1556752&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
(original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
Thu Jan  9 07:24:22 2014
@@ -17,6 +17,8 @@
 */
 package org.apache.hadoop.yarn.webapp.util;
 
+import static org.apache.hadoop.yarn.util.StringHelper.join;
+
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.net.UnknownHostException;
@@ -27,7 +29,9 @@ import org.apache.hadoop.conf.Configurat
 import org.apache.hadoop.http.HttpConfig;
 import org.apache.hadoop.http.HttpConfig.Policy;
 import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.util.ConverterUtils;
 
 import com.google.common.base.Joiner;
 
@@ -170,4 +174,11 @@ public class WebAppUtils {
       return schemePrefix + url;
     }
   }
+  
+  public static String getLogUrl(String nodeHttpAddress, String allocatedNode,
+      ContainerId containerId, String user) {
+    return join(HttpConfig.getSchemePrefix(), nodeHttpAddress, "/logs", "/",
+        allocatedNode, "/", ConverterUtils.toString(containerId), "/",
+        ConverterUtils.toString(containerId), "/", user);
+  }
 }

Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSController.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSController.java?rev=1556752&r1=1556751&r2=1556752&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSController.java
(original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSController.java
Thu Jan  9 07:24:22 2014
@@ -47,4 +47,10 @@ public class AHSController extends Contr
     render(ContainerPage.class);
   }
 
+  /**
+   * Render the logs page.
+   */
+  public void logs() {
+    render(AHSLogsPage.class);
+  }
 }

Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java?rev=1556752&r1=1556751&r2=1556752&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java
(original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java
Thu Jan  9 07:24:22 2014
@@ -45,5 +45,7 @@ public class AHSWebApp extends WebApp im
     route(pajoin("/appattempt", APPLICATION_ATTEMPT_ID), AHSController.class,
         "appattempt");
     route(pajoin("/container", CONTAINER_ID), AHSController.class, "container");
+    route(pajoin("/logs", NM_NODENAME, CONTAINER_ID, ENTITY_STRING, APP_OWNER,
+        CONTAINER_LOG_TYPE), AHSController.class, "logs");
   }
 }
\ No newline at end of file

Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java?rev=1556752&r1=1556751&r2=1556752&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
(original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
Thu Jan  9 07:24:22 2014
@@ -18,6 +18,7 @@
 package org.apache.hadoop.yarn.server.webapp;
 
 import static org.apache.hadoop.yarn.util.StringHelper.join;
+import static org.apache.hadoop.yarn.util.StringHelper.getPartUrl;
 import static org.apache.hadoop.yarn.webapp.YarnWebParams.APPLICATION_ATTEMPT_ID;
 
 import java.io.IOException;
@@ -126,6 +127,8 @@ public class AppAttemptBlock extends Htm
 
     StringBuilder containersTableData = new StringBuilder("[\n");
     for (ContainerReport containerReport : containers) {
+      String logURL = containerReport.getLogUrl();
+      logURL = getPartUrl(logURL, "log");
       ContainerInfo container = new ContainerInfo(containerReport);
       // ConatinerID numerical value parsed by parseHadoopID in yarn.dt.plugins.js
       containersTableData
@@ -141,9 +144,9 @@ public class AppAttemptBlock extends Htm
           .append("</a>\",\"")
           .append(container.getContainerExitStatus())
           .append("\",\"<a href='")
-          .append(container.getLogUrl() == null ? "#" : url(container.getLogUrl()))
+          .append(logURL == null ? "#" : url(logURL))
           .append("'>")
-          .append(container.getLogUrl() == null ? "N/A" : "Logs")
+          .append(logURL == null ? "N/A" : "Logs")
           .append("</a>\"],\n");
     }
     if (containersTableData.charAt(containersTableData.length() - 2) == ',') {

Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java?rev=1556752&r1=1556751&r2=1556752&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
(original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
Thu Jan  9 07:24:22 2014
@@ -19,6 +19,7 @@
 package org.apache.hadoop.yarn.server.webapp;
 
 import static org.apache.hadoop.yarn.util.StringHelper.join;
+import static org.apache.hadoop.yarn.util.StringHelper.getPartUrl;
 import static org.apache.hadoop.yarn.webapp.YarnWebParams.APPLICATION_ID;
 
 import java.io.IOException;
@@ -147,7 +148,8 @@ public class AppBlock extends HtmlBlock 
       if (containerReport != null) {
         ContainerInfo container = new ContainerInfo(containerReport);
         startTime = container.getStartedTime();
-        logsLink = container.getLogUrl();
+        logsLink = containerReport.getLogUrl();
+        logsLink = getPartUrl(logsLink,"log");
       }
       String nodeLink = null;
       if (appAttempt.getHost() != null && appAttempt.getRpcPort() >= 0

Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java?rev=1556752&r1=1556751&r2=1556752&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java
(original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java
Thu Jan  9 07:24:22 2014
@@ -18,6 +18,7 @@
 package org.apache.hadoop.yarn.server.webapp;
 
 import static org.apache.hadoop.yarn.util.StringHelper.join;
+import static org.apache.hadoop.yarn.util.StringHelper.getPartUrl;
 import static org.apache.hadoop.yarn.webapp.YarnWebParams.CONTAINER_ID;
 
 import java.io.IOException;
@@ -76,8 +77,10 @@ public class ContainerBlock extends Html
       puts("Container not found: " + containerid);
       return;
     }
+    
     ContainerInfo container = new ContainerInfo(containerReport);
-
+    String logURL = containerReport.getLogUrl();
+    logURL = getPartUrl(logURL,"log");
     setTitle(join("Container ", containerid));
 
     info("Container Overview").
@@ -91,8 +94,7 @@ public class ContainerBlock extends Html
             container.getFinishedTime()))).
     _("Resource:", container.getAllocatedMB() + " Memory, " +
         container.getAllocatedVCores() + " VCores").
-    _("Logs:", container.getLogUrl() == null ? "#" : root_url(container.getLogUrl()),
-        container.getLogUrl() == null ? "N/A" : container.getLogUrl()).
+    _("Logs:", logURL == null ? "#" : url(logURL), "Logs").
     _("Diagnostics:", container.getDiagnosticsInfo());
 
     html._(InfoBlock.class);

Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java?rev=1556752&r1=1556751&r2=1556752&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
(original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
Thu Jan  9 07:24:22 2014
@@ -18,8 +18,6 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.rmcontainer;
 
-import static org.apache.hadoop.yarn.util.StringHelper.join;
-
 import java.util.EnumSet;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
@@ -27,7 +25,6 @@ import java.util.concurrent.locks.Reentr
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.http.HttpConfig;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -46,7 +43,7 @@ import org.apache.hadoop.yarn.state.Inva
 import org.apache.hadoop.yarn.state.SingleArcTransition;
 import org.apache.hadoop.yarn.state.StateMachine;
 import org.apache.hadoop.yarn.state.StateMachineFactory;
-import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
 
 @SuppressWarnings({"unchecked", "rawtypes"})
 public class RMContainerImpl implements RMContainer {
@@ -366,10 +363,9 @@ public class RMContainerImpl implements 
     public void transition(RMContainerImpl container, RMContainerEvent event) {
       // The logs of running containers should be found on NM webUI
       // The logs should be accessible after the container is launched
-      container.logURL = join(HttpConfig.getSchemePrefix(),
-          container.container.getNodeHttpAddress(), "/node", "/containerlogs/",
-              ConverterUtils.toString(container.containerId), "/",
-              container.user);
+      container.logURL = WebAppUtils.getLogUrl(container.container
+          .getNodeHttpAddress(), container.getAllocatedNode().toString(),
+          container.containerId, container.user);
       // Unregister from containerAllocationExpirer.
       container.containerAllocationExpirer.unregister(container
           .getContainerId());



Mime
View raw message