Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 0E930200B36 for ; Wed, 22 Jun 2016 02:26:12 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 0CF20160A69; Wed, 22 Jun 2016 00:26:12 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 09F0C160A63 for ; Wed, 22 Jun 2016 02:26:10 +0200 (CEST) Received: (qmail 64192 invoked by uid 500); 22 Jun 2016 00:26:00 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 62109 invoked by uid 99); 22 Jun 2016 00:25:59 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 22 Jun 2016 00:25:59 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 88AB0EAD92; Wed, 22 Jun 2016 00:25:59 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: subru@apache.org To: common-commits@hadoop.apache.org Date: Wed, 22 Jun 2016 00:26:25 -0000 Message-Id: <6a9e4ab4369242fab184eda3253dbe4d@git.apache.org> In-Reply-To: <85a9aecfdfad441186f9f7f2137dd235@git.apache.org> References: <85a9aecfdfad441186f9f7f2137dd235@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [27/51] [abbrv] hadoop git commit: YARN-5251. Yarn CLI to obtain App logs for last 'n' bytes fails. Contributed by Xuan Gong. archived-at: Wed, 22 Jun 2016 00:26:12 -0000 YARN-5251. Yarn CLI to obtain App logs for last 'n' bytes fails. Contributed by Xuan Gong. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c35fa4a0 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c35fa4a0 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c35fa4a0 Branch: refs/heads/YARN-2915 Commit: c35fa4a0e57b99415e284c7c2f3b1a0bebc610c2 Parents: 09e82ac Author: Junping Du Authored: Fri Jun 17 08:24:24 2016 -0700 Committer: Junping Du Committed: Fri Jun 17 08:24:24 2016 -0700 ---------------------------------------------------------------------- .../hadoop/yarn/client/cli/TestLogsCLI.java | 2 +- .../logaggregation/AggregatedLogFormat.java | 26 +++++++++++--------- .../logaggregation/TestAggregatedLogFormat.java | 2 +- .../webapp/AHSWebServices.java | 11 +++++---- .../nodemanager/webapp/NMWebServices.java | 9 +++---- .../TestLogAggregationService.java | 2 +- 6 files changed, 27 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c35fa4a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java index df1c6b7..6683fc7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java @@ -400,7 +400,7 @@ public class TestLogsCLI { String logMessage = "Hello container_0_0001_01_000003 in stdout!"; int fileContentSize = logMessage.getBytes().length; - int tailContentSize = "End of LogType:syslog\n\n".getBytes().length; + int tailContentSize = "\nEnd of LogType:syslog\n\n".getBytes().length; // specify how many bytes we should get from logs // specify a position number, it would get the first n bytes from http://git-wip-us.apache.org/repos/asf/hadoop/blob/c35fa4a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java index 2fb18ed..7508dd5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java @@ -787,20 +787,19 @@ public class AggregatedLogFormat { long toSkip = 0; long totalBytesToRead = fileLength; + long skipAfterRead = 0; if (bytes < 0) { long absBytes = Math.abs(bytes); if (absBytes < fileLength) { toSkip = fileLength - absBytes; totalBytesToRead = absBytes; } - long skippedBytes = valueStream.skip(toSkip); - if (skippedBytes != toSkip) { - throw new IOException("The bytes were skipped are " - + "different from the caller requested"); - } + org.apache.hadoop.io.IOUtils.skipFully( + valueStream, toSkip); } else { if (bytes < fileLength) { totalBytesToRead = bytes; + skipAfterRead = fileLength - bytes; } } @@ -818,7 +817,9 @@ public class AggregatedLogFormat { pendingRead > buf.length ? buf.length : (int) pendingRead; len = valueStream.read(buf, 0, toRead); } - out.println("End of LogType:" + fileType); + org.apache.hadoop.io.IOUtils.skipFully( + valueStream, skipAfterRead); + out.println("\nEnd of LogType:" + fileType); out.println(""); } @@ -913,20 +914,19 @@ public class AggregatedLogFormat { long toSkip = 0; long totalBytesToRead = fileLength; + long skipAfterRead = 0; if (bytes < 0) { long absBytes = Math.abs(bytes); if (absBytes < fileLength) { toSkip = fileLength - absBytes; totalBytesToRead = absBytes; } - long skippedBytes = valueStream.skip(toSkip); - if (skippedBytes != toSkip) { - throw new IOException("The bytes were skipped are " - + "different from the caller requested"); - } + org.apache.hadoop.io.IOUtils.skipFully( + valueStream, toSkip); } else { if (bytes < fileLength) { totalBytesToRead = bytes; + skipAfterRead = fileLength - bytes; } } @@ -942,7 +942,9 @@ public class AggregatedLogFormat { toRead = pendingRead > buf.length ? buf.length : (int) pendingRead; len = valueStream.read(buf, 0, toRead); } - out.println("End of LogType:" + fileType); + org.apache.hadoop.io.IOUtils.skipFully( + valueStream, skipAfterRead); + out.println("\nEnd of LogType:" + fileType); out.println(""); return 0; } else { http://git-wip-us.apache.org/repos/asf/hadoop/blob/c35fa4a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java index 21e1655..45dd8ab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java @@ -258,7 +258,7 @@ public class TestAggregatedLogFormat { .currentTimeMillis())).length() : 0) + ("\nLogLength:" + numChars).length() + "\nLog Contents:\n".length() + numChars + "\n".length() - + "End of LogType:stdout\n".length(); + + "\nEnd of LogType:stdout\n".length(); Assert.assertTrue("LogType not matched", s.contains("LogType:stdout")); Assert.assertTrue("log file:stderr should not be aggregated.", !s.contains("LogType:stderr")); Assert.assertTrue("log file:logs should not be aggregated.", !s.contains("LogType:logs")); http://git-wip-us.apache.org/repos/asf/hadoop/blob/c35fa4a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java index 692b172..b56fae9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java @@ -404,20 +404,19 @@ public class AHSWebServices extends WebServices { long toSkip = 0; long totalBytesToRead = fileLength; + long skipAfterRead = 0; if (bytes < 0) { long absBytes = Math.abs(bytes); if (absBytes < fileLength) { toSkip = fileLength - absBytes; totalBytesToRead = absBytes; } - long skippedBytes = valueStream.skip(toSkip); - if (skippedBytes != toSkip) { - throw new IOException("The bytes were skipped are " - + "different from the caller requested"); - } + org.apache.hadoop.io.IOUtils.skipFully( + valueStream, toSkip); } else { if (bytes < fileLength) { totalBytesToRead = bytes; + skipAfterRead = fileLength - bytes; } } @@ -435,6 +434,8 @@ public class AHSWebServices extends WebServices { : (int) pendingRead; len = valueStream.read(buf, 0, toRead); } + org.apache.hadoop.io.IOUtils.skipFully( + valueStream, skipAfterRead); sb = new StringBuilder(); sb.append("\nEnd of LogType:" + fileType + "\n"); b = sb.toString().getBytes(Charset.forName("UTF-8")); http://git-wip-us.apache.org/repos/asf/hadoop/blob/c35fa4a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java index 3a30392..4e38e90 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java @@ -264,20 +264,18 @@ public class NMWebServices { byte[] buf = new byte[bufferSize]; long toSkip = 0; long totalBytesToRead = fileLength; + long skipAfterRead = 0; if (bytes < 0) { long absBytes = Math.abs(bytes); if (absBytes < fileLength) { toSkip = fileLength - absBytes; totalBytesToRead = absBytes; } - long skippedBytes = fis.skip(toSkip); - if (skippedBytes != toSkip) { - throw new IOException("The bytes were skipped are different " - + "from the caller requested"); - } + org.apache.hadoop.io.IOUtils.skipFully(fis, toSkip); } else { if (bytes < fileLength) { totalBytesToRead = bytes; + skipAfterRead = fileLength - bytes; } } @@ -295,6 +293,7 @@ public class NMWebServices { : (int) pendingRead; len = fis.read(buf, 0, toRead); } + org.apache.hadoop.io.IOUtils.skipFully(fis, skipAfterRead); os.flush(); } finally { IOUtils.closeQuietly(fis); http://git-wip-us.apache.org/repos/asf/hadoop/blob/c35fa4a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java index c98e366..1edb841 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java @@ -948,7 +948,7 @@ public class TestLogAggregationService extends BaseContainerManagerTest { Assert.assertEquals(numOfLogsPerContainer, thisContainerMap.size()); for (String fileType : logFiles) { String expectedValue = - containerStr + " Hello " + fileType + "!End of LogType:" + containerStr + " Hello " + fileType + "!\nEnd of LogType:" + fileType; LOG.info("Expected log-content : " + new String(expectedValue)); String foundValue = thisContainerMap.remove(fileType); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org