Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id BA335200B25 for ; Wed, 8 Jun 2016 20:23:37 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id B7759160A35; Wed, 8 Jun 2016 18:23:37 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 150E2160A0E for ; Wed, 8 Jun 2016 20:23:35 +0200 (CEST) Received: (qmail 52571 invoked by uid 500); 8 Jun 2016 18:23:35 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 52557 invoked by uid 99); 8 Jun 2016 18:23:35 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 08 Jun 2016 18:23:35 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 120F7DFC4F; Wed, 8 Jun 2016 18:23:35 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: xgong@apache.org To: common-commits@hadoop.apache.org Message-Id: <5cabcdfe7dbe412b9c99f2b7f2807b32@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: Revert "YARN-4920. ATS/NM should support a link to dowload/get the logs in text format. Contributed by Xuan Gong." Date: Wed, 8 Jun 2016 18:23:35 +0000 (UTC) archived-at: Wed, 08 Jun 2016 18:23:37 -0000 Repository: hadoop Updated Branches: refs/heads/branch-2.8 593d2c66c -> 11b4d1e48 Revert "YARN-4920. ATS/NM should support a link to dowload/get the logs in text format. Contributed by Xuan Gong." This reverts commit 22ac37615a933f9cee8cf19ad0182586a037b690. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/11b4d1e4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/11b4d1e4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/11b4d1e4 Branch: refs/heads/branch-2.8 Commit: 11b4d1e486486da9c59ecf11322a06f63db9b3e0 Parents: 593d2c6 Author: Xuan Authored: Wed Jun 8 11:23:12 2016 -0700 Committer: Xuan Committed: Wed Jun 8 11:23:12 2016 -0700 ---------------------------------------------------------------------- .../webapp/AHSWebServices.java | 267 +------------------ ...pplicationHistoryManagerOnTimelineStore.java | 29 +- .../webapp/TestAHSWebServices.java | 203 +------------- .../yarn/server/webapp/dao/ContainerInfo.java | 6 - .../nodemanager/webapp/NMWebServices.java | 22 +- .../nodemanager/webapp/TestNMWebServices.java | 12 +- 6 files changed, 17 insertions(+), 522 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/11b4d1e4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java index deae894..2f65494 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java @@ -18,11 +18,6 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice.webapp; -import java.io.DataInputStream; -import java.io.EOFException; -import java.io.IOException; -import java.io.OutputStream; -import java.nio.charset.Charset; import java.util.Collections; import java.util.Set; @@ -33,30 +28,13 @@ import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; -import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import javax.ws.rs.core.StreamingOutput; -import javax.ws.rs.core.Response.ResponseBuilder; -import javax.ws.rs.core.Response.Status; -import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileContext; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.ApplicationBaseProtocol; import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat; -import org.apache.hadoop.yarn.logaggregation.LogAggregationUtils; -import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey; import org.apache.hadoop.yarn.server.webapp.WebServices; import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptsInfo; @@ -64,10 +42,9 @@ import org.apache.hadoop.yarn.server.webapp.dao.AppInfo; import org.apache.hadoop.yarn.server.webapp.dao.AppsInfo; import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo; import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo; -import org.apache.hadoop.yarn.util.Times; import org.apache.hadoop.yarn.util.timeline.TimelineUtils; import org.apache.hadoop.yarn.webapp.BadRequestException; -import com.google.common.base.Joiner; + import com.google.inject.Inject; import com.google.inject.Singleton; @@ -75,17 +52,9 @@ import com.google.inject.Singleton; @Path("/ws/v1/applicationhistory") public class AHSWebServices extends WebServices { - private static final String NM_DOWNLOAD_URI_STR = - "/ws/v1/node/containerlogs"; - private static final Joiner JOINER = Joiner.on(""); - private static final Joiner DOT_JOINER = Joiner.on(". "); - private final Configuration conf; - @Inject - public AHSWebServices(ApplicationBaseProtocol appBaseProt, - Configuration conf) { + public AHSWebServices(ApplicationBaseProtocol appBaseProt) { super(appBaseProt); - this.conf = conf; } @GET @@ -203,236 +172,4 @@ public class AHSWebServices extends WebServices { } } } - - @GET - @Path("/containerlogs/{containerid}/{filename}") - @Produces({ MediaType.TEXT_PLAIN }) - @Public - @Unstable - public Response getLogs(@Context HttpServletRequest req, - @Context HttpServletResponse res, - @PathParam("containerid") String containerIdStr, - @PathParam("filename") String filename, - @QueryParam("download") String download) { - init(res); - ContainerId containerId; - try { - containerId = ContainerId.fromString(containerIdStr); - } catch (IllegalArgumentException ex) { - return createBadResponse(Status.NOT_FOUND, - "Invalid ContainerId: " + containerIdStr); - } - - boolean downloadFile = parseBooleanParam(download); - - ApplicationId appId = containerId.getApplicationAttemptId() - .getApplicationId(); - AppInfo appInfo; - try { - appInfo = super.getApp(req, res, appId.toString()); - } catch (Exception ex) { - // directly find logs from HDFS. - return sendStreamOutputResponse(appId, null, null, containerIdStr, - filename, downloadFile); - } - String appOwner = appInfo.getUser(); - - ContainerInfo containerInfo; - try { - containerInfo = super.getContainer( - req, res, appId.toString(), - containerId.getApplicationAttemptId().toString(), - containerId.toString()); - } catch (Exception ex) { - if (isFinishedState(appInfo.getAppState())) { - // directly find logs from HDFS. - return sendStreamOutputResponse(appId, appOwner, null, containerIdStr, - filename, downloadFile); - } - return createBadResponse(Status.INTERNAL_SERVER_ERROR, - "Can not get ContainerInfo for the container: " + containerId); - } - String nodeId = containerInfo.getNodeId(); - if (isRunningState(appInfo.getAppState())) { - String nodeHttpAddress = containerInfo.getNodeHttpAddress(); - String uri = "/" + containerId.toString() + "/" + filename; - String resURI = JOINER.join(nodeHttpAddress, NM_DOWNLOAD_URI_STR, uri); - String query = req.getQueryString(); - if (query != null && !query.isEmpty()) { - resURI += "?" + query; - } - ResponseBuilder response = Response.status( - HttpServletResponse.SC_TEMPORARY_REDIRECT); - response.header("Location", resURI); - return response.build(); - } else if (isFinishedState(appInfo.getAppState())) { - return sendStreamOutputResponse(appId, appOwner, nodeId, - containerIdStr, filename, downloadFile); - } else { - return createBadResponse(Status.NOT_FOUND, - "The application is not at Running or Finished State."); - } - } - - private boolean isRunningState(YarnApplicationState appState) { - return appState == YarnApplicationState.RUNNING; - } - - private boolean isFinishedState(YarnApplicationState appState) { - return appState == YarnApplicationState.FINISHED - || appState == YarnApplicationState.FAILED - || appState == YarnApplicationState.KILLED; - } - - private Response createBadResponse(Status status, String errMessage) { - Response response = Response.status(status) - .entity(DOT_JOINER.join(status.toString(), errMessage)).build(); - return response; - } - - private boolean parseBooleanParam(String param) { - return ("true").equalsIgnoreCase(param); - } - - private Response sendStreamOutputResponse(ApplicationId appId, - String appOwner, String nodeId, String containerIdStr, - String fileName, boolean downloadFile) { - StreamingOutput stream = null; - try { - stream = getStreamingOutput(appId, appOwner, nodeId, - containerIdStr, fileName); - } catch (Exception ex) { - return createBadResponse(Status.INTERNAL_SERVER_ERROR, - ex.getMessage()); - } - if (stream == null) { - return createBadResponse(Status.INTERNAL_SERVER_ERROR, - "Can not get log for container: " + containerIdStr); - } - ResponseBuilder response = Response.ok(stream); - if (downloadFile) { - response.header("Content-Type", "application/octet-stream"); - response.header("Content-Disposition", "attachment; filename=" - + fileName); - } - return response.build(); - } - - private StreamingOutput getStreamingOutput(ApplicationId appId, - String appOwner, final String nodeId, final String containerIdStr, - final String logFile) throws IOException{ - String suffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(conf); - org.apache.hadoop.fs.Path remoteRootLogDir = new org.apache.hadoop.fs.Path( - conf.get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, - YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR)); - org.apache.hadoop.fs.Path qualifiedRemoteRootLogDir = - FileContext.getFileContext(conf).makeQualified(remoteRootLogDir); - FileContext fc = FileContext.getFileContext( - qualifiedRemoteRootLogDir.toUri(), conf); - org.apache.hadoop.fs.Path remoteAppDir = null; - if (appOwner == null) { - org.apache.hadoop.fs.Path toMatch = LogAggregationUtils - .getRemoteAppLogDir(remoteRootLogDir, appId, "*", suffix); - FileStatus[] matching = fc.util().globStatus(toMatch); - if (matching == null || matching.length != 1) { - return null; - } - remoteAppDir = matching[0].getPath(); - } else { - remoteAppDir = LogAggregationUtils - .getRemoteAppLogDir(remoteRootLogDir, appId, appOwner, suffix); - } - final RemoteIterator nodeFiles; - nodeFiles = fc.listStatus(remoteAppDir); - if (!nodeFiles.hasNext()) { - return null; - } - - StreamingOutput stream = new StreamingOutput() { - - @Override - public void write(OutputStream os) throws IOException, - WebApplicationException { - byte[] buf = new byte[65535]; - boolean findLogs = false; - while (nodeFiles.hasNext()) { - final FileStatus thisNodeFile = nodeFiles.next(); - String nodeName = thisNodeFile.getPath().getName(); - if ((nodeId == null || nodeName.contains(LogAggregationUtils - .getNodeString(nodeId))) && !nodeName.endsWith( - LogAggregationUtils.TMP_FILE_SUFFIX)) { - AggregatedLogFormat.LogReader reader = - new AggregatedLogFormat.LogReader(conf, - thisNodeFile.getPath()); - DataInputStream valueStream; - LogKey key = new LogKey(); - valueStream = reader.next(key); - while (valueStream != null && !key.toString() - .equals(containerIdStr)) { - // Next container - key = new LogKey(); - valueStream = reader.next(key); - } - if (valueStream == null) { - continue; - } - while (true) { - try { - String fileType = valueStream.readUTF(); - String fileLengthStr = valueStream.readUTF(); - long fileLength = Long.parseLong(fileLengthStr); - if (fileType.equalsIgnoreCase(logFile)) { - StringBuilder sb = new StringBuilder(); - sb.append("LogType:"); - sb.append(fileType + "\n"); - sb.append("Log Upload Time:"); - sb.append(Times.format(System.currentTimeMillis()) + "\n"); - sb.append("LogLength:"); - sb.append(fileLengthStr + "\n"); - sb.append("Log Contents:\n"); - byte[] b = sb.toString().getBytes(Charset.forName("UTF-8")); - os.write(b, 0, b.length); - - long curRead = 0; - long pendingRead = fileLength - curRead; - int toRead = pendingRead > buf.length ? buf.length - : (int) pendingRead; - int len = valueStream.read(buf, 0, toRead); - while (len != -1 && curRead < fileLength) { - os.write(buf, 0, len); - curRead += len; - - pendingRead = fileLength - curRead; - toRead = pendingRead > buf.length ? buf.length - : (int) pendingRead; - len = valueStream.read(buf, 0, toRead); - } - sb = new StringBuilder(); - sb.append("\nEnd of LogType:" + fileType + "\n"); - b = sb.toString().getBytes(Charset.forName("UTF-8")); - os.write(b, 0, b.length); - findLogs = true; - } else { - long totalSkipped = 0; - long currSkipped = 0; - while (currSkipped != -1 && totalSkipped < fileLength) { - currSkipped = valueStream.skip(fileLength - totalSkipped); - totalSkipped += currSkipped; - } - } - } catch (EOFException eof) { - break; - } - } - } - } - os.flush(); - if (!findLogs) { - throw new IOException("Can not find logs for container:" - + containerIdStr); - } - } - }; - return stream; - } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/11b4d1e4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java index b65b22b..06f6ae3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java @@ -80,11 +80,9 @@ public class TestApplicationHistoryManagerOnTimelineStore { store = createStore(SCALE); TimelineEntities entities = new TimelineEntities(); entities.addEntity(createApplicationTimelineEntity( - ApplicationId.newInstance(0, SCALE + 1), true, true, false, false, - YarnApplicationState.FINISHED)); + ApplicationId.newInstance(0, SCALE + 1), true, true, false, false)); entities.addEntity(createApplicationTimelineEntity( - ApplicationId.newInstance(0, SCALE + 2), true, false, true, false, - YarnApplicationState.FINISHED)); + ApplicationId.newInstance(0, SCALE + 2), true, false, true, false)); store.put(entities); } @@ -142,10 +140,10 @@ public class TestApplicationHistoryManagerOnTimelineStore { ApplicationId appId = ApplicationId.newInstance(0, i); if (i == 2) { entities.addEntity(createApplicationTimelineEntity( - appId, true, false, false, true, YarnApplicationState.FINISHED)); + appId, true, false, false, true)); } else { entities.addEntity(createApplicationTimelineEntity( - appId, false, false, false, false, YarnApplicationState.FINISHED)); + appId, false, false, false, false)); } store.put(entities); for (int j = 1; j <= scale; ++j) { @@ -162,16 +160,6 @@ public class TestApplicationHistoryManagerOnTimelineStore { } } } - TimelineEntities entities = new TimelineEntities(); - ApplicationId appId = ApplicationId.newInstance(1234, 1); - ApplicationAttemptId appAttemptId = - ApplicationAttemptId.newInstance(appId, 1); - ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1); - entities.addEntity(createApplicationTimelineEntity( - appId, true, false, false, false, YarnApplicationState.RUNNING)); - entities.addEntity(createAppAttemptTimelineEntity(appAttemptId)); - entities.addEntity(createContainerEntity(containerId)); - store.put(entities); } @Test @@ -367,7 +355,7 @@ public class TestApplicationHistoryManagerOnTimelineStore { historyManager.getApplications(Long.MAX_VALUE, 0L, Long.MAX_VALUE) .values(); Assert.assertNotNull(apps); - Assert.assertEquals(SCALE + 2, apps.size()); + Assert.assertEquals(SCALE + 1, apps.size()); ApplicationId ignoredAppId = ApplicationId.newInstance(0, SCALE + 2); for (ApplicationReport app : apps) { Assert.assertNotEquals(ignoredAppId, app.getApplicationId()); @@ -479,8 +467,7 @@ public class TestApplicationHistoryManagerOnTimelineStore { private static TimelineEntity createApplicationTimelineEntity( ApplicationId appId, boolean emptyACLs, boolean noAttemptId, - boolean wrongAppId, boolean enableUpdateEvent, - YarnApplicationState state) { + boolean wrongAppId, boolean enableUpdateEvent) { TimelineEntity entity = new TimelineEntity(); entity.setEntityType(ApplicationMetricsConstants.ENTITY_TYPE); if (wrongAppId) { @@ -530,7 +517,7 @@ public class TestApplicationHistoryManagerOnTimelineStore { eventInfo.put(ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO, FinalApplicationStatus.UNDEFINED.toString()); eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO, - state.toString()); + YarnApplicationState.FINISHED.toString()); if (!noAttemptId) { eventInfo.put(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO, ApplicationAttemptId.newInstance(appId, 1)); @@ -635,8 +622,6 @@ public class TestApplicationHistoryManagerOnTimelineStore { entityInfo.put(ContainerMetricsConstants.ALLOCATED_PORT_ENTITY_INFO, 100); entityInfo .put(ContainerMetricsConstants.ALLOCATED_PRIORITY_ENTITY_INFO, -1); - entityInfo.put(ContainerMetricsConstants - .ALLOCATED_HOST_HTTP_ADDRESS_ENTITY_INFO, "http://test:1234"); entity.setOtherInfo(entityInfo); TimelineEvent tEvent = new TimelineEvent(); tEvent.setEventType(ContainerMetricsConstants.CREATED_EVENT_TYPE); http://git-wip-us.apache.org/repos/asf/hadoop/blob/11b4d1e4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java index f985fe4..20dfe45 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java @@ -19,30 +19,17 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice.webapp; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import java.io.File; -import java.io.FileWriter; -import java.io.Writer; -import java.net.HttpURLConnection; -import java.net.URI; -import java.net.URL; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.List; import java.util.Properties; import javax.servlet.FilterConfig; import javax.servlet.ServletException; -import javax.servlet.http.HttpServletResponse; import javax.ws.rs.core.MediaType; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authentication.server.AuthenticationFilter; import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler; import org.apache.hadoop.yarn.api.ApplicationBaseProtocol; @@ -55,8 +42,6 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat; -import org.apache.hadoop.yarn.logaggregation.LogAggregationUtils; import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryClientService; import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryManagerOnTimelineStore; import org.apache.hadoop.yarn.server.applicationhistoryservice.TestApplicationHistoryManagerOnTimelineStore; @@ -96,17 +81,12 @@ import com.sun.jersey.test.framework.WebAppDescriptor; public class TestAHSWebServices extends JerseyTestBase { private static ApplicationHistoryClientService historyClientService; - private static AHSWebServices ahsWebservice; private static final String[] USERS = new String[] { "foo" , "bar" }; private static final int MAX_APPS = 5; - private static Configuration conf; - private static FileSystem fs; - private static final String remoteLogRootDir = "target/logs/"; - private static final String rootLogDir = "target/LocalLogs"; @BeforeClass public static void setupClass() throws Exception { - conf = new YarnConfiguration(); + Configuration conf = new YarnConfiguration(); TimelineStore store = TestApplicationHistoryManagerOnTimelineStore.createStore(MAX_APPS); TimelineACLsManager aclsManager = new TimelineACLsManager(conf); @@ -115,8 +95,6 @@ public class TestAHSWebServices extends JerseyTestBase { new TimelineDataManager(store, aclsManager); conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); conf.set(YarnConfiguration.YARN_ADMIN_ACL, "foo"); - conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true); - conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir); dataManager.init(conf); ApplicationACLsManager appAclsManager = new ApplicationACLsManager(conf); ApplicationHistoryManagerOnTimelineStore historyManager = @@ -130,8 +108,6 @@ public class TestAHSWebServices extends JerseyTestBase { }; historyClientService.init(conf); historyClientService.start(); - ahsWebservice = new AHSWebServices(historyClientService, conf); - fs = FileSystem.get(conf); } @AfterClass @@ -139,8 +115,6 @@ public class TestAHSWebServices extends JerseyTestBase { if (historyClientService != null) { historyClientService.stop(); } - fs.delete(new Path(remoteLogRootDir), true); - fs.delete(new Path(rootLogDir), true); } @Parameterized.Parameters @@ -153,7 +127,7 @@ public class TestAHSWebServices extends JerseyTestBase { @Override protected void configureServlets() { bind(JAXBContextResolver.class); - bind(AHSWebServices.class).toInstance(ahsWebservice);; + bind(AHSWebServices.class); bind(GenericExceptionHandler.class); bind(ApplicationBaseProtocol.class).toInstance(historyClientService); serve("/*").with(GuiceContainer.class); @@ -497,177 +471,4 @@ public class TestAHSWebServices extends JerseyTestBase { assertEquals(ContainerState.COMPLETE.toString(), container.getString("containerState")); } - - @Test(timeout = 10000) - public void testContainerLogsForFinishedApps() throws Exception { - String fileName = "syslog"; - String user = "user1"; - UserGroupInformation ugi = UserGroupInformation.createRemoteUser("user1"); - NodeId nodeId = NodeId.newInstance("test host", 100); - NodeId nodeId2 = NodeId.newInstance("host2", 1234); - //prepare the logs for remote directory - ApplicationId appId = ApplicationId.newInstance(0, 1); - // create local logs - List rootLogDirList = new ArrayList(); - rootLogDirList.add(rootLogDir); - Path rootLogDirPath = new Path(rootLogDir); - if (fs.exists(rootLogDirPath)) { - fs.delete(rootLogDirPath, true); - } - assertTrue(fs.mkdirs(rootLogDirPath)); - - Path appLogsDir = new Path(rootLogDirPath, appId.toString()); - if (fs.exists(appLogsDir)) { - fs.delete(appLogsDir, true); - } - assertTrue(fs.mkdirs(appLogsDir)); - - // create container logs in local log file dir - // create two container log files. We can get containerInfo - // for container1 from AHS, but can not get such info for - // container100 - ApplicationAttemptId appAttemptId = - ApplicationAttemptId.newInstance(appId, 1); - ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 1); - ContainerId containerId100 = ContainerId.newContainerId(appAttemptId, 100); - createContainerLogInLocalDir(appLogsDir, containerId1, fs, fileName, - ("Hello." + containerId1)); - createContainerLogInLocalDir(appLogsDir, containerId100, fs, fileName, - ("Hello." + containerId100)); - - // upload container logs to remote log dir - Path path = new Path(conf.get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR) + - user + "/logs/" + appId.toString()); - if (fs.exists(path)) { - fs.delete(path, true); - } - assertTrue(fs.mkdirs(path)); - uploadContainerLogIntoRemoteDir(ugi, conf, rootLogDirList, nodeId, - containerId1, path, fs); - uploadContainerLogIntoRemoteDir(ugi, conf, rootLogDirList, nodeId2, - containerId100, path, fs); - - // test whether we can find container log from remote diretory if - // the containerInfo for this container could be fetched from AHS. - WebResource r = resource(); - ClientResponse response = r.path("ws").path("v1") - .path("applicationhistory").path("containerlogs") - .path(containerId1.toString()).path(fileName) - .queryParam("user.name", user) - .accept(MediaType.TEXT_PLAIN) - .get(ClientResponse.class); - String responseText = response.getEntity(String.class); - assertTrue(responseText.contains("Hello." + containerId1)); - - // test whether we can find container log from remote diretory if - // the containerInfo for this container could not be fetched from AHS. - r = resource(); - response = r.path("ws").path("v1") - .path("applicationhistory").path("containerlogs") - .path(containerId100.toString()).path(fileName) - .queryParam("user.name", user) - .accept(MediaType.TEXT_PLAIN) - .get(ClientResponse.class); - responseText = response.getEntity(String.class); - assertTrue(responseText.contains("Hello." + containerId100)); - - // create an application which can not be found from AHS - ApplicationId appId100 = ApplicationId.newInstance(0, 100); - appLogsDir = new Path(rootLogDirPath, appId100.toString()); - if (fs.exists(appLogsDir)) { - fs.delete(appLogsDir, true); - } - assertTrue(fs.mkdirs(appLogsDir)); - ApplicationAttemptId appAttemptId100 = - ApplicationAttemptId.newInstance(appId100, 1); - ContainerId containerId1ForApp100 = ContainerId - .newContainerId(appAttemptId100, 1); - createContainerLogInLocalDir(appLogsDir, containerId1ForApp100, fs, - fileName, ("Hello." + containerId1ForApp100)); - path = new Path(conf.get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR) + - user + "/logs/" + appId100.toString()); - if (fs.exists(path)) { - fs.delete(path, true); - } - assertTrue(fs.mkdirs(path)); - uploadContainerLogIntoRemoteDir(ugi, conf, rootLogDirList, nodeId2, - containerId1ForApp100, path, fs); - r = resource(); - response = r.path("ws").path("v1") - .path("applicationhistory").path("containerlogs") - .path(containerId1ForApp100.toString()).path(fileName) - .queryParam("user.name", user) - .accept(MediaType.TEXT_PLAIN) - .get(ClientResponse.class); - responseText = response.getEntity(String.class); - assertTrue(responseText.contains("Hello." + containerId1ForApp100)); - } - - private static void createContainerLogInLocalDir(Path appLogsDir, - ContainerId containerId, FileSystem fs, String fileName, String content) - throws Exception { - Path containerLogsDir = new Path(appLogsDir, containerId.toString()); - if (fs.exists(containerLogsDir)) { - fs.delete(containerLogsDir, true); - } - assertTrue(fs.mkdirs(containerLogsDir)); - Writer writer = - new FileWriter(new File(containerLogsDir.toString(), fileName)); - writer.write(content); - writer.close(); - } - - private static void uploadContainerLogIntoRemoteDir(UserGroupInformation ugi, - Configuration configuration, List rootLogDirs, NodeId nodeId, - ContainerId containerId, Path appDir, FileSystem fs) throws Exception { - Path path = - new Path(appDir, LogAggregationUtils.getNodeString(nodeId)); - AggregatedLogFormat.LogWriter writer = - new AggregatedLogFormat.LogWriter(configuration, path, ugi); - writer.writeApplicationOwner(ugi.getUserName()); - - writer.append(new AggregatedLogFormat.LogKey(containerId), - new AggregatedLogFormat.LogValue(rootLogDirs, containerId, - ugi.getShortUserName())); - writer.close(); - } - - @Test(timeout = 10000) - public void testContainerLogsForRunningApps() throws Exception { - String fileName = "syslog"; - String user = "user1"; - ApplicationId appId = ApplicationId.newInstance( - 1234, 1); - ApplicationAttemptId appAttemptId = - ApplicationAttemptId.newInstance(appId, 1); - ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 1); - WebResource r = resource(); - URI requestURI = r.path("ws").path("v1") - .path("applicationhistory").path("containerlogs") - .path(containerId1.toString()).path(fileName) - .queryParam("user.name", user).getURI(); - String redirectURL = getRedirectURL(requestURI.toString()); - assertTrue(redirectURL != null); - assertTrue(redirectURL.contains("test:1234")); - assertTrue(redirectURL.contains("ws/v1/node/containerlogs")); - assertTrue(redirectURL.contains(containerId1.toString())); - assertTrue(redirectURL.contains("user.name=" + user)); - } - - private static String getRedirectURL(String url) { - String redirectUrl = null; - try { - HttpURLConnection conn = (HttpURLConnection) new URL(url) - .openConnection(); - // do not automatically follow the redirection - // otherwise we get too many redirections exception - conn.setInstanceFollowRedirects(false); - if(conn.getResponseCode() == HttpServletResponse.SC_TEMPORARY_REDIRECT) { - redirectUrl = conn.getHeaderField("Location"); - } - } catch (Exception e) { - // throw new RuntimeException(e); - } - return redirectUrl; - } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/11b4d1e4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java index 1a5ee85..936c5d4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java @@ -48,7 +48,6 @@ public class ContainerInfo { protected int containerExitStatus; protected ContainerState containerState; protected String nodeHttpAddress; - protected String nodeId; public ContainerInfo() { // JAXB needs this @@ -72,7 +71,6 @@ public class ContainerInfo { containerExitStatus = container.getContainerExitStatus(); containerState = container.getContainerState(); nodeHttpAddress = container.getNodeHttpAddress(); - nodeId = container.getAssignedNode().toString(); } public String getContainerId() { @@ -126,8 +124,4 @@ public class ContainerInfo { public String getNodeHttpAddress() { return nodeHttpAddress; } - - public String getNodeId() { - return nodeId; - } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/11b4d1e4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java index 57e729c..fddeb04 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java @@ -33,7 +33,6 @@ import javax.ws.rs.QueryParam; import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; -import javax.ws.rs.core.Response.ResponseBuilder; import javax.ws.rs.core.Response.Status; import javax.ws.rs.core.StreamingOutput; import javax.ws.rs.core.UriInfo; @@ -216,8 +215,7 @@ public class NMWebServices { @Public @Unstable public Response getLogs(@PathParam("containerid") String containerIdStr, - @PathParam("filename") String filename, - @QueryParam("download") String download) { + @PathParam("filename") String filename) { ContainerId containerId; try { containerId = ConverterUtils.toContainerId(containerIdStr); @@ -234,7 +232,7 @@ public class NMWebServices { } catch (YarnException ex) { return Response.serverError().entity(ex.getMessage()).build(); } - boolean downloadFile = parseBooleanParam(download); + try { final FileInputStream fis = ContainerLogsUtils.openLogFileForRead( containerIdStr, logFile, nmContext); @@ -252,22 +250,10 @@ public class NMWebServices { os.flush(); } }; - ResponseBuilder resp = Response.ok(stream); - if (downloadFile) { - resp.header("Content-Type", "application/octet-stream"); - resp.header("Content-Disposition", "attachment; filename=" - + logFile.getName()); - } - return resp.build(); + + return Response.ok(stream).build(); } catch (IOException ex) { return Response.serverError().entity(ex.getMessage()).build(); } } - - private boolean parseBooleanParam(String param) { - if (param != null) { - return ("true").equalsIgnoreCase(param); - } - return false; - } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/11b4d1e4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java index c10d4c8..1f5590c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java @@ -50,6 +50,7 @@ import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService; import org.apache.hadoop.yarn.server.nodemanager.NodeManager; import org.apache.hadoop.yarn.server.nodemanager.ResourceView; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationImpl; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState; import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch; import org.apache.hadoop.yarn.server.nodemanager.webapp.WebServer.NMWebApp; @@ -351,16 +352,7 @@ public class TestNMWebServices extends JerseyTestBase { .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); String responseText = response.getEntity(String.class); assertEquals(logMessage, responseText); - - // ask and download it - response = r.path("ws").path("v1").path("node").path("containerlogs") - .path(containerIdStr).path(filename).queryParam("download", "true") - .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); - responseText = response.getEntity(String.class); - assertEquals(logMessage, responseText); - assertEquals(200, response.getStatus()); - assertEquals("application/octet-stream", response.getType().toString()); - + // ask for file that doesn't exist response = r.path("ws").path("v1").path("node") .path("containerlogs").path(containerIdStr).path("uhhh") --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org