Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 764E5200B5B for ; Fri, 5 Aug 2016 19:56:46 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 74C46160A8E; Fri, 5 Aug 2016 17:56:46 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 1C44E160A64 for ; Fri, 5 Aug 2016 19:56:44 +0200 (CEST) Received: (qmail 33137 invoked by uid 500); 5 Aug 2016 17:56:44 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 33127 invoked by uid 99); 5 Aug 2016 17:56:44 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 05 Aug 2016 17:56:44 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 1B6D3EC22C; Fri, 5 Aug 2016 17:56:44 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: junping_du@apache.org To: common-commits@hadoop.apache.org Message-Id: <85893dd98a894e5c890a64eabbd1f699@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: YARN-5470. Differentiate exactly match with regex in yarn log CLI. Contributed by Xuan Gong. Date: Fri, 5 Aug 2016 17:56:44 +0000 (UTC) archived-at: Fri, 05 Aug 2016 17:56:46 -0000 Repository: hadoop Updated Branches: refs/heads/trunk 3f100d76f -> e605d47df YARN-5470. Differentiate exactly match with regex in yarn log CLI. Contributed by Xuan Gong. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e605d47d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e605d47d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e605d47d Branch: refs/heads/trunk Commit: e605d47df05619c6b1c18aca59f709899498da75 Parents: 3f100d7 Author: Junping Du Authored: Fri Aug 5 10:54:36 2016 -0700 Committer: Junping Du Committed: Fri Aug 5 10:55:29 2016 -0700 ---------------------------------------------------------------------- .../apache/hadoop/yarn/client/cli/LogsCLI.java | 125 +++++++++++-------- .../hadoop/yarn/client/cli/TestLogsCLI.java | 69 +++++++--- .../logaggregation/ContainerLogsRequest.java | 10 +- .../yarn/logaggregation/LogCLIHelpers.java | 8 +- 4 files changed, 137 insertions(+), 75 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e605d47d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java index c49d463..ec045b2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java @@ -25,6 +25,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -90,6 +91,7 @@ public class LogsCLI extends Configured implements Tool { = "show_container_log_info"; private static final String OUT_OPTION = "out"; private static final String SIZE_OPTION = "size"; + private static final String REGEX_OPTION = "regex"; public static final String HELP_CMD = "help"; private PrintStream outStream = System.out; private YarnClient yarnClient = null; @@ -126,6 +128,7 @@ public class LogsCLI extends Configured implements Tool { boolean nodesList = false; boolean showApplicationLogInfo = false; boolean showContainerLogInfo = false; + boolean useRegex = false; String[] logFiles = null; List amContainersList = new ArrayList(); String localDir = null; @@ -142,6 +145,7 @@ public class LogsCLI extends Configured implements Tool { showApplicationLogInfo = commandLine.hasOption( SHOW_APPLICATION_LOG_INFO); showContainerLogInfo = commandLine.hasOption(SHOW_CONTAINER_LOG_INFO); + useRegex = commandLine.hasOption(REGEX_OPTION); if (getAMContainerLogs) { try { amContainersList = parseAMContainer(commandLine, printOpts); @@ -243,11 +247,11 @@ public class LogsCLI extends Configured implements Tool { } } - List logs = new ArrayList(); - if (fetchAllLogFiles(logFiles)) { - logs.add(".*"); + Set logs = new HashSet(); + if (fetchAllLogFiles(logFiles, useRegex)) { + logs.add("ALL"); } else if (logFiles != null && logFiles.length > 0) { - logs = Arrays.asList(logFiles); + logs.addAll(Arrays.asList(logFiles)); } ContainerLogsRequest request = new ContainerLogsRequest(appId, @@ -268,15 +272,15 @@ public class LogsCLI extends Configured implements Tool { // To get am logs if (getAMContainerLogs) { return fetchAMContainerLogs(request, amContainersList, - logCliHelper); + logCliHelper, useRegex); } int resultCode = 0; if (containerIdStr != null) { - return fetchContainerLogs(request, logCliHelper); + return fetchContainerLogs(request, logCliHelper, useRegex); } else { if (nodeAddress == null) { - resultCode = fetchApplicationLogs(request, logCliHelper); + resultCode = fetchApplicationLogs(request, logCliHelper, useRegex); } else { System.err.println("Should at least provide ContainerId!"); printHelpMessage(printOpts); @@ -362,7 +366,7 @@ public class LogsCLI extends Configured implements Tool { return amContainersList; } - private boolean fetchAllLogFiles(String[] logFiles) { + private boolean fetchAllLogFiles(String[] logFiles, boolean useRegex) { // If no value is specified for the PER_CONTAINER_LOG_FILES_OPTION option, // we will assume all logs. @@ -371,7 +375,8 @@ public class LogsCLI extends Configured implements Tool { } List logs = Arrays.asList(logFiles); - if (logs.contains("ALL") || logs.contains(".*")) { + if (logs.contains("ALL") || logs.contains("*")|| + (logs.contains(".*") && useRegex)) { return true; } @@ -417,8 +422,8 @@ public class LogsCLI extends Configured implements Tool { @Private @VisibleForTesting public int printContainerLogsFromRunningApplication(Configuration conf, - ContainerLogsRequest request, LogCLIHelpers logCliHelper) - throws IOException { + ContainerLogsRequest request, LogCLIHelpers logCliHelper, + boolean useRegex) throws IOException { String containerIdStr = request.getContainerId().toString(); String localDir = request.getOutputLocalDir(); String nodeHttpAddress = request.getNodeHttpAddress(); @@ -441,7 +446,8 @@ public class LogsCLI extends Configured implements Tool { for (PerLogFileInfo fileInfo : allLogFileInfos) { fileNames.add(fileInfo.getFileName()); } - List matchedFiles = getMatchedLogFiles(request, fileNames); + Set matchedFiles = getMatchedLogFiles(request, fileNames, + useRegex); if (matchedFiles.isEmpty()) { System.err.println("Can not find any log file matching the pattern: " + request.getLogTypes() + " for the container: " + containerIdStr @@ -503,10 +509,10 @@ public class LogsCLI extends Configured implements Tool { } private int printContainerLogsForFinishedApplication( - ContainerLogsRequest request, LogCLIHelpers logCliHelper) - throws IOException { + ContainerLogsRequest request, LogCLIHelpers logCliHelper, + boolean useRegex) throws IOException { ContainerLogsRequest newOptions = getMatchedLogOptions( - request, logCliHelper); + request, logCliHelper, useRegex); if (newOptions == null) { System.err.println("Can not find any log file matching the pattern: " + request.getLogTypes() + " for the container: " @@ -518,10 +524,10 @@ public class LogsCLI extends Configured implements Tool { } private int printContainerLogsForFinishedApplicationWithoutNodeId( - ContainerLogsRequest request, LogCLIHelpers logCliHelper) - throws IOException { + ContainerLogsRequest request, LogCLIHelpers logCliHelper, + boolean useRegex) throws IOException { ContainerLogsRequest newOptions = getMatchedLogOptions( - request, logCliHelper); + request, logCliHelper, useRegex); if (newOptions == null) { System.err.println("Can not find any log file matching the pattern: " + request.getLogTypes() + " for the container: " @@ -549,7 +555,7 @@ public class LogsCLI extends Configured implements Tool { private int printAMContainerLogs(Configuration conf, ContainerLogsRequest request, List amContainers, - LogCLIHelpers logCliHelper) throws Exception { + LogCLIHelpers logCliHelper, boolean useRegex) throws Exception { List amContainersList = null; List requests = new ArrayList(); @@ -613,7 +619,7 @@ public class LogsCLI extends Configured implements Tool { if (amContainers.contains("ALL")) { for (ContainerLogsRequest amRequest : requests) { - outputAMContainerLogs(amRequest, conf, logCliHelper); + outputAMContainerLogs(amRequest, conf, logCliHelper, useRegex); } outStream.println(); outStream.println("Specified ALL for -am option. " @@ -623,11 +629,11 @@ public class LogsCLI extends Configured implements Tool { int amContainerId = Integer.parseInt(amContainer.trim()); if (amContainerId == -1) { outputAMContainerLogs(requests.get(requests.size() - 1), conf, - logCliHelper); + logCliHelper, useRegex); } else { if (amContainerId <= requests.size()) { outputAMContainerLogs(requests.get(amContainerId - 1), conf, - logCliHelper); + logCliHelper, useRegex); } else { System.err.println(String.format("ERROR: Specified AM containerId" + " (%s) exceeds the number of AM containers (%s).", @@ -641,7 +647,8 @@ public class LogsCLI extends Configured implements Tool { } private void outputAMContainerLogs(ContainerLogsRequest request, - Configuration conf, LogCLIHelpers logCliHelper) throws Exception { + Configuration conf, LogCLIHelpers logCliHelper, boolean useRegex) + throws Exception { String nodeHttpAddress = request.getNodeHttpAddress(); String containerId = request.getContainerId(); String nodeId = request.getNodeId(); @@ -650,10 +657,10 @@ public class LogsCLI extends Configured implements Tool { if (containerId != null && !containerId.isEmpty()) { if (nodeId != null && !nodeId.isEmpty()) { printContainerLogsForFinishedApplication(request, - logCliHelper); + logCliHelper, useRegex); } else { printContainerLogsForFinishedApplicationWithoutNodeId( - request, logCliHelper); + request, logCliHelper, useRegex); } } } else { @@ -663,7 +670,7 @@ public class LogsCLI extends Configured implements Tool { .getContainerState(); request.setContainerState(containerState); printContainerLogsFromRunningApplication(conf, - request, logCliHelper); + request, logCliHelper, useRegex); } } } @@ -746,12 +753,15 @@ public class LogsCLI extends Configured implements Tool { opts.addOption(amOption); Option logFileOpt = new Option(PER_CONTAINER_LOG_FILES_OPTION, true, "Specify comma-separated value " - + "to get specified container log files. Use \"ALL\" to fetch all the " - + "log files for the container. It also supports Java Regex."); + + "to get exact matched log files. Use \"ALL\" or \"*\"to " + + "fetch all the log files for the container. Specific -regex " + + "for using java regex to find matched log files."); logFileOpt.setValueSeparator(','); logFileOpt.setArgs(Option.UNLIMITED_VALUES); logFileOpt.setArgName("Log File Name"); opts.addOption(logFileOpt); + opts.addOption(REGEX_OPTION, false, "Work with -log_files to find " + + "matched files by using java regex."); opts.addOption(SHOW_CONTAINER_LOG_INFO, false, "Show the container log metadata, " + "including log-file names, the size of the log files. " @@ -794,6 +804,7 @@ public class LogsCLI extends Configured implements Tool { printOpts.addOption(commandOpts.getOption(SHOW_CONTAINER_LOG_INFO)); printOpts.addOption(commandOpts.getOption(OUT_OPTION)); printOpts.addOption(commandOpts.getOption(SIZE_OPTION)); + printOpts.addOption(commandOpts.getOption(REGEX_OPTION)); return printOpts; } @@ -829,14 +840,14 @@ public class LogsCLI extends Configured implements Tool { } private int fetchAMContainerLogs(ContainerLogsRequest request, - List amContainersList, LogCLIHelpers logCliHelper) - throws Exception { + List amContainersList, LogCLIHelpers logCliHelper, + boolean useRegex) throws Exception { return printAMContainerLogs(getConf(), request, amContainersList, - logCliHelper); + logCliHelper, useRegex); } private int fetchContainerLogs(ContainerLogsRequest request, - LogCLIHelpers logCliHelper) throws IOException { + LogCLIHelpers logCliHelper, boolean useRegex) throws IOException { int resultCode = 0; String appIdStr = request.getAppId().toString(); String containerIdStr = request.getContainerId(); @@ -850,10 +861,10 @@ public class LogsCLI extends Configured implements Tool { // to logCliHelper so that it fetches all the logs if (nodeAddress != null && !nodeAddress.isEmpty()) { return printContainerLogsForFinishedApplication( - request, logCliHelper); + request, logCliHelper, useRegex); } else { return printContainerLogsForFinishedApplicationWithoutNodeId( - request, logCliHelper); + request, logCliHelper, useRegex); } } String nodeHttpAddress = null; @@ -875,7 +886,7 @@ public class LogsCLI extends Configured implements Tool { } catch (IOException | YarnException ex) { if (isAppFinished) { return printContainerLogsForFinishedApplicationWithoutNodeId( - request, logCliHelper); + request, logCliHelper, useRegex); } else { System.err.println("Unable to get logs for this container:" + containerIdStr + "for the application:" + appIdStr @@ -892,18 +903,19 @@ public class LogsCLI extends Configured implements Tool { // by calling NodeManager webservice. if (!isAppFinished) { resultCode = printContainerLogsFromRunningApplication(getConf(), request, - logCliHelper); + logCliHelper, useRegex); } else { // If the application is in the final state, we will directly // get the container logs from HDFS. resultCode = printContainerLogsForFinishedApplication( - request, logCliHelper); + request, logCliHelper, useRegex); } return resultCode; } private int fetchApplicationLogs(ContainerLogsRequest options, - LogCLIHelpers logCliHelper) throws IOException, YarnException { + LogCLIHelpers logCliHelper, boolean useRegex) throws IOException, + YarnException { // If the application has finished, we would fetch the logs // from HDFS. // If the application is still running, we would get the full @@ -912,7 +924,7 @@ public class LogsCLI extends Configured implements Tool { int resultCode = -1; if (options.isAppFinished()) { ContainerLogsRequest newOptions = getMatchedLogOptions( - options, logCliHelper); + options, logCliHelper, useRegex); if (newOptions == null) { System.err.println("Can not find any log file matching the pattern: " + options.getLogTypes() + " for the application: " @@ -926,7 +938,7 @@ public class LogsCLI extends Configured implements Tool { getContainersLogRequestForRunningApplication(options); for (ContainerLogsRequest container : containerLogRequests) { int result = printContainerLogsFromRunningApplication(getConf(), - container, logCliHelper); + container, logCliHelper, useRegex); if (result == 0) { resultCode = 0; } @@ -955,14 +967,14 @@ public class LogsCLI extends Configured implements Tool { } private ContainerLogsRequest getMatchedLogOptions( - ContainerLogsRequest request, LogCLIHelpers logCliHelper) - throws IOException { + ContainerLogsRequest request, LogCLIHelpers logCliHelper, + boolean useRegex) throws IOException { ContainerLogsRequest newOptions = new ContainerLogsRequest(request); if (request.getLogTypes() != null && !request.getLogTypes().isEmpty()) { - List matchedFiles = new ArrayList(); - if (!request.getLogTypes().contains(".*")) { + Set matchedFiles = new HashSet(); + if (!request.getLogTypes().contains("ALL")) { Set files = logCliHelper.listContainerLogs(request); - matchedFiles = getMatchedLogFiles(request, files); + matchedFiles = getMatchedLogFiles(request, files, useRegex); if (matchedFiles.isEmpty()) { return null; } @@ -972,20 +984,29 @@ public class LogsCLI extends Configured implements Tool { return newOptions; } - private List getMatchedLogFiles(ContainerLogsRequest options, - Collection candidate) throws IOException { - List matchedFiles = new ArrayList(); - List filePattern = options.getLogTypes(); + private Set getMatchedLogFiles(ContainerLogsRequest options, + Collection candidate, boolean useRegex) throws IOException { + Set matchedFiles = new HashSet(); + Set filePattern = options.getLogTypes(); + if (options.getLogTypes().contains("ALL")) { + return new HashSet(candidate); + } for (String file : candidate) { - if (isFileMatching(file, filePattern)) { - matchedFiles.add(file); + if (useRegex) { + if (isFileMatching(file, filePattern)) { + matchedFiles.add(file); + } + } else { + if (filePattern.contains(file)) { + matchedFiles.add(file); + } } } return matchedFiles; } private boolean isFileMatching(String fileType, - List logTypes) { + Set logTypes) { for (String logType : logTypes) { Pattern filterPattern = Pattern.compile(logType); boolean match = filterPattern.matcher(fileType).find(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/e605d47d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java index 329d856..467b204 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyBoolean; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; @@ -46,7 +47,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; - +import java.util.Set; import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; @@ -211,14 +212,17 @@ public class TestLogsCLI { pw.println(" aggregated logs. This option can only be"); pw.println(" used with finished applications."); pw.println(" -log_files Specify comma-separated value to get"); - pw.println(" specified container log files. Use \"ALL\""); - pw.println(" to fetch all the log files for the"); - pw.println(" container. It also supports Java Regex."); + pw.println(" exact matched log files. Use \"ALL\" or"); + pw.println(" \"*\"to fetch all the log files for the"); + pw.println(" container. Specific -regex for using java"); + pw.println(" regex to find matched log files."); pw.println(" -nodeAddress NodeAddress in the format nodename:port"); pw.println(" -out Local directory for storing individual"); pw.println(" container logs. The container logs will"); pw.println(" be stored based on the node the container"); pw.println(" ran on."); + pw.println(" -regex Work with -log_files to find matched"); + pw.println(" files by using java regex."); pw.println(" -show_application_log_info Show the containerIds which belong to the"); pw.println(" specific Application. You can combine"); pw.println(" this with --nodeAddress to get"); @@ -286,6 +290,7 @@ public class TestLogsCLI { // create two logs for container3 in localLogDir logTypes.add("stdout"); + logTypes.add("stdout1234"); createContainerLogInLocalDir(appLogsDir, containerId3, fs, logTypes); Path path = @@ -336,10 +341,27 @@ public class TestLogsCLI { logMessage(containerId3, "syslog"))); assertTrue(sysOutStream.toString().contains( logMessage(containerId3, "stdout"))); + assertTrue(sysOutStream.toString().contains( + logMessage(containerId3, "stdout1234"))); + sysOutStream.reset(); + + exitCode = cli.run(new String[] {"-applicationId", appId.toString(), + "-log_files", ".*", "-regex"}); + assertTrue(exitCode == 0); + assertTrue(sysOutStream.toString().contains( + logMessage(containerId1, "syslog"))); + assertTrue(sysOutStream.toString().contains( + logMessage(containerId2, "syslog"))); + assertTrue(sysOutStream.toString().contains( + logMessage(containerId3, "syslog"))); + assertTrue(sysOutStream.toString().contains( + logMessage(containerId3, "stdout"))); + assertTrue(sysOutStream.toString().contains( + logMessage(containerId3, "stdout1234"))); sysOutStream.reset(); exitCode = cli.run(new String[] {"-applicationId", appId.toString(), - "-log_files", ".*"}); + "-log_files", "*"}); assertTrue(exitCode == 0); assertTrue(sysOutStream.toString().contains( logMessage(containerId1, "syslog"))); @@ -349,11 +371,13 @@ public class TestLogsCLI { logMessage(containerId3, "syslog"))); assertTrue(sysOutStream.toString().contains( logMessage(containerId3, "stdout"))); + assertTrue(sysOutStream.toString().contains( + logMessage(containerId3, "stdout1234"))); int fullSize = sysOutStream.toByteArray().length; sysOutStream.reset(); exitCode = cli.run(new String[] {"-applicationId", appId.toString(), - "-log_files", "std*"}); + "-log_files", "stdout"}); assertTrue(exitCode == 0); assertFalse(sysOutStream.toString().contains( logMessage(containerId1, "syslog"))); @@ -363,6 +387,23 @@ public class TestLogsCLI { logMessage(containerId3, "syslog"))); assertTrue(sysOutStream.toString().contains( logMessage(containerId3, "stdout"))); + assertFalse(sysOutStream.toString().contains( + logMessage(containerId3, "stdout1234"))); + sysOutStream.reset(); + + exitCode = cli.run(new String[] {"-applicationId", appId.toString(), + "-log_files", "std*", "-regex"}); + assertTrue(exitCode == 0); + assertFalse(sysOutStream.toString().contains( + logMessage(containerId1, "syslog"))); + assertFalse(sysOutStream.toString().contains( + logMessage(containerId2, "syslog"))); + assertFalse(sysOutStream.toString().contains( + logMessage(containerId3, "syslog"))); + assertTrue(sysOutStream.toString().contains( + logMessage(containerId3, "stdout"))); + assertTrue(sysOutStream.toString().contains( + logMessage(containerId3, "stdout1234"))); sysOutStream.reset(); exitCode = cli.run(new String[] {"-applicationId", appId.toString(), @@ -376,7 +417,7 @@ public class TestLogsCLI { // specify the bytes which is larger than the actual file size, // we would get the full logs exitCode = cli.run(new String[] {"-applicationId", appId.toString(), - "-log_files", ".*", "-size", "10000" }); + "-log_files", "*", "-size", "10000" }); assertTrue(exitCode == 0); assertTrue(sysOutStream.toByteArray().length == fullSize); sysOutStream.reset(); @@ -568,7 +609,7 @@ public class TestLogsCLI { LogsCLI cli = spy(new LogsCLIForTest(mockYarnClient)); doReturn(0).when(cli).printContainerLogsFromRunningApplication( any(Configuration.class), any(ContainerLogsRequest.class), - any(LogCLIHelpers.class)); + any(LogCLIHelpers.class), anyBoolean()); cli.setConf(new YarnConfiguration()); int exitCode = cli.run(new String[] {"-applicationId", appId.toString()}); @@ -581,16 +622,16 @@ public class TestLogsCLI { // printContainerLogsFromRunningApplication twice verify(cli, times(2)).printContainerLogsFromRunningApplication( any(Configuration.class), logsRequestCaptor.capture(), - any(LogCLIHelpers.class)); + any(LogCLIHelpers.class), anyBoolean()); - // Verify that the log-type is * + // Verify that the log-type is "ALL" List capturedRequests = logsRequestCaptor.getAllValues(); Assert.assertEquals(2, capturedRequests.size()); - List logTypes0 = capturedRequests.get(0).getLogTypes(); - List logTypes1 = capturedRequests.get(1).getLogTypes(); - Assert.assertEquals(".*", logTypes0.get(0)); - Assert.assertEquals(".*", logTypes1.get(0)); + Set logTypes0 = capturedRequests.get(0).getLogTypes(); + Set logTypes1 = capturedRequests.get(1).getLogTypes(); + Assert.assertTrue(logTypes0.contains("ALL") && (logTypes0.size() == 1)); + Assert.assertTrue(logTypes1.contains("ALL") && (logTypes1.size() == 1)); } @Test (timeout = 15000) http://git-wip-us.apache.org/repos/asf/hadoop/blob/e605d47d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/ContainerLogsRequest.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/ContainerLogsRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/ContainerLogsRequest.java index af65fef..30aeb6c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/ContainerLogsRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/ContainerLogsRequest.java @@ -18,7 +18,7 @@ package org.apache.hadoop.yarn.logaggregation; -import java.util.List; +import java.util.Set; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerState; @@ -30,7 +30,7 @@ public class ContainerLogsRequest { private String appOwner; private boolean appFinished; private String outputLocalDir; - private List logTypes; + private Set logTypes; private long bytes; private ContainerState containerState; @@ -52,7 +52,7 @@ public class ContainerLogsRequest { public ContainerLogsRequest(ApplicationId applicationId, boolean isAppFinished, String owner, String address, String httpAddress, String container, String localDir, - List logs, long bytes, ContainerState containerState) { + Set logs, long bytes, ContainerState containerState) { this.setAppId(applicationId); this.setAppFinished(isAppFinished); this.setAppOwner(owner); @@ -121,11 +121,11 @@ public class ContainerLogsRequest { this.outputLocalDir = outputLocalDir; } - public List getLogTypes() { + public Set getLogTypes() { return logTypes; } - public void setLogTypes(List logTypes) { + public void setLogTypes(Set logTypes) { this.logTypes = logTypes; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e605d47d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java index 13934d2..3ea3fa6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java @@ -67,7 +67,7 @@ public class LogCLIHelpers implements Configurable { options.setContainerId(containerId); options.setNodeId(nodeId); options.setAppOwner(jobOwner); - List logs = new ArrayList(); + Set logs = new HashSet(); options.setLogTypes(logs); options.setBytes(Long.MAX_VALUE); return dumpAContainerLogsForLogType(options, false); @@ -136,7 +136,7 @@ public class LogCLIHelpers implements Configurable { String nodeId = options.getNodeId(); String containerId = options.getContainerId(); String localDir = options.getOutputLocalDir(); - List logType = options.getLogTypes(); + List logType = new ArrayList(options.getLogTypes()); RemoteIterator nodeFiles = getRemoteNodeFileDir( applicationId, jobOwner); if (nodeFiles == null) { @@ -208,7 +208,7 @@ public class LogCLIHelpers implements Configurable { String jobOwner = options.getAppOwner(); String containerId = options.getContainerId(); String localDir = options.getOutputLocalDir(); - List logType = options.getLogTypes(); + List logType = new ArrayList(options.getLogTypes()); RemoteIterator nodeFiles = getRemoteNodeFileDir( applicationId, jobOwner); if (nodeFiles == null) { @@ -344,7 +344,7 @@ public class LogCLIHelpers implements Configurable { ApplicationId appId = options.getAppId(); String appOwner = options.getAppOwner(); String localDir = options.getOutputLocalDir(); - List logTypes = options.getLogTypes(); + List logTypes = new ArrayList(options.getLogTypes()); RemoteIterator nodeFiles = getRemoteNodeFileDir( appId, appOwner); if (nodeFiles == null) { --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org