Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id C0EA2200B43 for ; Tue, 19 Jul 2016 23:41:15 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id BFC09160A8E; Tue, 19 Jul 2016 21:41:15 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id B6E85160A5C for ; Tue, 19 Jul 2016 23:41:14 +0200 (CEST) Received: (qmail 19185 invoked by uid 500); 19 Jul 2016 21:41:03 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 17853 invoked by uid 99); 19 Jul 2016 21:41:02 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 19 Jul 2016 21:41:02 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 55814E1893; Tue, 19 Jul 2016 21:41:02 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: subru@apache.org To: common-commits@hadoop.apache.org Date: Tue, 19 Jul 2016 21:41:43 -0000 Message-Id: <2b707b52aa8a41c1bc4d7e752cf57ada@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [43/50] hadoop git commit: YARN-5213. Fix a bug in LogCLIHelpers which cause TestLogsCLI#testFetchApplictionLogs fails intermittently. Contributed by Xuan Gong. archived-at: Tue, 19 Jul 2016 21:41:15 -0000 YARN-5213. Fix a bug in LogCLIHelpers which cause TestLogsCLI#testFetchApplictionLogs fails intermittently. Contributed by Xuan Gong. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6e0b3858 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6e0b3858 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6e0b3858 Branch: refs/heads/YARN-2915 Commit: 6e0b38589dc401027cb6aae93bd21d3db74b3116 Parents: 82c4d7d Author: Junping Du Authored: Tue Jul 19 07:17:58 2016 -0700 Committer: Subru Krishnan Committed: Tue Jul 19 14:39:32 2016 -0700 ---------------------------------------------------------------------- .../hadoop/yarn/client/cli/TestLogsCLI.java | 66 +++++++++++--------- .../yarn/logaggregation/LogCLIHelpers.java | 11 ++++ 2 files changed, 46 insertions(+), 31 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e0b3858/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java index bda489f..329d856 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java @@ -329,26 +329,26 @@ public class TestLogsCLI { int exitCode = cli.run(new String[] { "-applicationId", appId.toString() }); assertTrue(exitCode == 0); assertTrue(sysOutStream.toString().contains( - "Hello container_0_0001_01_000001 in syslog!")); + logMessage(containerId1, "syslog"))); assertTrue(sysOutStream.toString().contains( - "Hello container_0_0001_01_000002 in syslog!")); + logMessage(containerId2, "syslog"))); assertTrue(sysOutStream.toString().contains( - "Hello container_0_0001_01_000003 in syslog!")); + logMessage(containerId3, "syslog"))); assertTrue(sysOutStream.toString().contains( - "Hello container_0_0001_01_000003 in stdout!")); + logMessage(containerId3, "stdout"))); sysOutStream.reset(); exitCode = cli.run(new String[] {"-applicationId", appId.toString(), "-log_files", ".*"}); assertTrue(exitCode == 0); assertTrue(sysOutStream.toString().contains( - "Hello container_0_0001_01_000001 in syslog!")); + logMessage(containerId1, "syslog"))); assertTrue(sysOutStream.toString().contains( - "Hello container_0_0001_01_000002 in syslog!")); + logMessage(containerId2, "syslog"))); assertTrue(sysOutStream.toString().contains( - "Hello container_0_0001_01_000003 in syslog!")); + logMessage(containerId3, "syslog"))); assertTrue(sysOutStream.toString().contains( - "Hello container_0_0001_01_000003 in stdout!")); + logMessage(containerId3, "stdout"))); int fullSize = sysOutStream.toByteArray().length; sysOutStream.reset(); @@ -356,13 +356,13 @@ public class TestLogsCLI { "-log_files", "std*"}); assertTrue(exitCode == 0); assertFalse(sysOutStream.toString().contains( - "Hello container_0_0001_01_000001 in syslog!")); + logMessage(containerId1, "syslog"))); assertFalse(sysOutStream.toString().contains( - "Hello container_0_0001_01_000002 in syslog!")); + logMessage(containerId2, "syslog"))); assertFalse(sysOutStream.toString().contains( - "Hello container_0_0001_01_000003 in syslog!")); + logMessage(containerId3, "syslog"))); assertTrue(sysOutStream.toString().contains( - "Hello container_0_0001_01_000003 in stdout!")); + logMessage(containerId3, "stdout"))); sysOutStream.reset(); exitCode = cli.run(new String[] {"-applicationId", appId.toString(), @@ -390,7 +390,7 @@ public class TestLogsCLI { containerId1.toString() }); assertTrue(exitCode == 0); assertTrue(sysOutStream.toString().contains( - "Hello container_0_0001_01_000001 in syslog!")); + logMessage(containerId1, "syslog"))); assertTrue(sysOutStream.toString().contains("Log Upload Time")); assertTrue(!sysOutStream.toString().contains( "Logs for container " + containerId1.toString() @@ -413,9 +413,9 @@ public class TestLogsCLI { String fullContext = sysOutStream.toString(); sysOutStream.reset(); - String logMessage = "Hello container_0_0001_01_000003 in stdout!"; + String logMessage = logMessage(containerId3, "stdout"); int fileContentSize = logMessage.getBytes().length; - int tailContentSize = "\nEnd of LogType:syslog\n\n".getBytes().length; + int tailContentSize = "\nEnd of LogType:stdout\n\n".getBytes().length; // specify how many bytes we should get from logs // specify a position number, it would get the first n bytes from @@ -470,9 +470,9 @@ public class TestLogsCLI { containerId3.toString() }); assertTrue(exitCode == 0); assertTrue(sysOutStream.toString().contains( - "Hello container_0_0001_01_000003 in syslog!")); + logMessage(containerId3, "syslog"))); assertTrue(sysOutStream.toString().contains( - "Hello container_0_0001_01_000003 in stdout!")); + logMessage(containerId3, "stdout"))); sysOutStream.reset(); // set -log_files option as stdout @@ -483,9 +483,9 @@ public class TestLogsCLI { containerId3.toString() , "-log_files", "stdout"}); assertTrue(exitCode == 0); assertTrue(sysOutStream.toString().contains( - "Hello container_0_0001_01_000003 in stdout!")); + logMessage(containerId3, "stdout"))); assertTrue(!sysOutStream.toString().contains( - "Hello container_0_0001_01_000003 in syslog!")); + logMessage(containerId3, "syslog"))); sysOutStream.reset(); YarnClient mockYarnClientWithException = @@ -498,9 +498,9 @@ public class TestLogsCLI { "-containerId", containerId3.toString() }); assertTrue(exitCode == 0); assertTrue(sysOutStream.toString().contains( - "Hello container_0_0001_01_000003 in syslog!")); + logMessage(containerId3, "syslog"))); assertTrue(sysOutStream.toString().contains( - "Hello container_0_0001_01_000003 in stdout!")); + logMessage(containerId3, "stdout"))); assertTrue(sysOutStream.toString().contains( containerId3 + " on " + LogAggregationUtils.getNodeString(nodeId))); sysOutStream.reset(); @@ -510,9 +510,9 @@ public class TestLogsCLI { cli.run(new String[] { "-containerId", containerId3.toString() }); assertTrue(exitCode == 0); assertTrue(sysOutStream.toString().contains( - "Hello container_0_0001_01_000003 in syslog!")); + logMessage(containerId3, "syslog"))); assertTrue(sysOutStream.toString().contains( - "Hello container_0_0001_01_000003 in stdout!")); + logMessage(containerId3, "stdout"))); assertTrue(sysOutStream.toString().contains( containerId3 + " on " + LogAggregationUtils.getNodeString(nodeId))); sysOutStream.reset(); @@ -662,7 +662,7 @@ public class TestLogsCLI { "-appOwner", testUser}); assertTrue(exitCode == 0); assertTrue(sysOutStream.toString().contains( - "Hello " + containerId + " in syslog!")); + logMessage(containerId, "syslog"))); sysOutStream.reset(); // Verify that we can not get the application logs @@ -684,8 +684,8 @@ public class TestLogsCLI { exitCode = cli.run(new String[] { "-applicationId", appId.toString()}); assertTrue(exitCode == 0); - assertTrue(sysOutStream.toString().contains("Hello " - + containerId + " in syslog!")); + assertTrue(sysOutStream.toString().contains( + logMessage(containerId, "syslog"))); sysOutStream.reset(); // Verify that we could get the err message "Can not find the appOwner" @@ -847,16 +847,14 @@ public class TestLogsCLI { assertTrue(container1Dir[0].getPath().getName().equals( containerId1.toString())); String container1= readContainerContent(container1Dir[0].getPath(), fs); - assertTrue(container1.contains("Hello " + containerId1 - + " in syslog!")); + assertTrue(container1.contains(logMessage(containerId1, "syslog"))); FileStatus[] container2Dir = fs.listStatus(nodeDir[1].getPath()); assertTrue(container2Dir.length == 1); assertTrue(container2Dir[0].getPath().getName().equals( containerId2.toString())); String container2= readContainerContent(container2Dir[0].getPath(), fs); - assertTrue(container2.contains("Hello " + containerId2 - + " in syslog!")); + assertTrue(container2.contains(logMessage(containerId2, "syslog"))); } finally { fs.delete(new Path(remoteLogRootDir), true); fs.delete(new Path(rootLogDir), true); @@ -1139,11 +1137,17 @@ public class TestLogsCLI { for (String logType : logTypes) { Writer writer = new FileWriter(new File(containerLogsDir.toString(), logType)); - writer.write("Hello " + containerId + " in " + logType + "!"); + writer.write(logMessage(containerId, logType)); writer.close(); } } + private static String logMessage(ContainerId containerId, String logType) { + StringBuilder sb = new StringBuilder(); + sb.append("Hello " + containerId + " in " + logType + "!"); + return sb.toString(); + } + private static void uploadContainerLogIntoRemoteDir(UserGroupInformation ugi, Configuration configuration, List rootLogDirs, NodeId nodeId, ContainerId containerId, Path appDir, FileSystem fs) throws Exception { http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e0b3858/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java index ef56504..13934d2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java @@ -157,10 +157,18 @@ public class LogCLIHelpers implements Configurable { AggregatedLogFormat.LogReader reader = null; PrintStream out = createPrintStream(localDir, fileName, containerId); try { + reader = new AggregatedLogFormat.LogReader(getConf(), + thisNodeFile.getPath()); + if (getContainerLogsStream(containerId, reader) == null) { + continue; + } String containerString = String.format(CONTAINER_ON_NODE_PATTERN, containerId, thisNodeFile.getPath().getName()); out.println(containerString); out.println(StringUtils.repeat("=", containerString.length())); + // We have to re-create reader object to reset the stream index + // after calling getContainerLogsStream which would move the stream + // index to the end of the log file. reader = new AggregatedLogFormat.LogReader(getConf(), thisNodeFile.getPath()); @@ -220,6 +228,9 @@ public class LogCLIHelpers implements Configurable { if (getContainerLogsStream(containerId, reader) == null) { continue; } + // We have to re-create reader object to reset the stream index + // after calling getContainerLogsStream which would move the stream + // index to the end of the log file. reader = new AggregatedLogFormat.LogReader(getConf(), thisNodeFile.getPath()); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org