hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From junping...@apache.org
Subject hadoop git commit: YARN-5251. Yarn CLI to obtain App logs for last 'n' bytes fails. Contributed by Xuan Gong.
Date Fri, 17 Jun 2016 15:21:00 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk 09e82acaf -> c35fa4a0e


YARN-5251. Yarn CLI to obtain App logs for last 'n' bytes fails. Contributed by Xuan Gong.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c35fa4a0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c35fa4a0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c35fa4a0

Branch: refs/heads/trunk
Commit: c35fa4a0e57b99415e284c7c2f3b1a0bebc610c2
Parents: 09e82ac
Author: Junping Du <junping_du@apache.org>
Authored: Fri Jun 17 08:24:24 2016 -0700
Committer: Junping Du <junping_du@apache.org>
Committed: Fri Jun 17 08:24:24 2016 -0700

----------------------------------------------------------------------
 .../hadoop/yarn/client/cli/TestLogsCLI.java     |  2 +-
 .../logaggregation/AggregatedLogFormat.java     | 26 +++++++++++---------
 .../logaggregation/TestAggregatedLogFormat.java |  2 +-
 .../webapp/AHSWebServices.java                  | 11 +++++----
 .../nodemanager/webapp/NMWebServices.java       |  9 +++----
 .../TestLogAggregationService.java              |  2 +-
 6 files changed, 27 insertions(+), 25 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c35fa4a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
index df1c6b7..6683fc7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
@@ -400,7 +400,7 @@ public class TestLogsCLI {
 
     String logMessage = "Hello container_0_0001_01_000003 in stdout!";
     int fileContentSize = logMessage.getBytes().length;
-    int tailContentSize = "End of LogType:syslog\n\n".getBytes().length;
+    int tailContentSize = "\nEnd of LogType:syslog\n\n".getBytes().length;
 
     // specify how many bytes we should get from logs
     // specify a position number, it would get the first n bytes from

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c35fa4a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
index 2fb18ed..7508dd5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
@@ -787,20 +787,19 @@ public class AggregatedLogFormat {
 
       long toSkip = 0;
       long totalBytesToRead = fileLength;
+      long skipAfterRead = 0;
       if (bytes < 0) {
         long absBytes = Math.abs(bytes);
         if (absBytes < fileLength) {
           toSkip = fileLength - absBytes;
           totalBytesToRead = absBytes;
         }
-        long skippedBytes = valueStream.skip(toSkip);
-        if (skippedBytes != toSkip) {
-          throw new IOException("The bytes were skipped are "
-              + "different from the caller requested");
-        }
+        org.apache.hadoop.io.IOUtils.skipFully(
+            valueStream, toSkip);
       } else {
         if (bytes < fileLength) {
           totalBytesToRead = bytes;
+          skipAfterRead = fileLength - bytes;
         }
       }
 
@@ -818,7 +817,9 @@ public class AggregatedLogFormat {
                   pendingRead > buf.length ? buf.length : (int) pendingRead;
         len = valueStream.read(buf, 0, toRead);
       }
-      out.println("End of LogType:" + fileType);
+      org.apache.hadoop.io.IOUtils.skipFully(
+          valueStream, skipAfterRead);
+      out.println("\nEnd of LogType:" + fileType);
       out.println("");
     }
 
@@ -913,20 +914,19 @@ public class AggregatedLogFormat {
 
         long toSkip = 0;
         long totalBytesToRead = fileLength;
+        long skipAfterRead = 0;
         if (bytes < 0) {
           long absBytes = Math.abs(bytes);
           if (absBytes < fileLength) {
             toSkip = fileLength - absBytes;
             totalBytesToRead = absBytes;
           }
-          long skippedBytes = valueStream.skip(toSkip);
-          if (skippedBytes != toSkip) {
-            throw new IOException("The bytes were skipped are "
-                + "different from the caller requested");
-          }
+          org.apache.hadoop.io.IOUtils.skipFully(
+              valueStream, toSkip);
         } else {
           if (bytes < fileLength) {
             totalBytesToRead = bytes;
+            skipAfterRead = fileLength - bytes;
           }
         }
 
@@ -942,7 +942,9 @@ public class AggregatedLogFormat {
           toRead = pendingRead > buf.length ? buf.length : (int) pendingRead;
           len = valueStream.read(buf, 0, toRead);
         }
-        out.println("End of LogType:" + fileType);
+        org.apache.hadoop.io.IOUtils.skipFully(
+            valueStream, skipAfterRead);
+        out.println("\nEnd of LogType:" + fileType);
         out.println("");
         return 0;
       } else {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c35fa4a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
index 21e1655..45dd8ab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
@@ -258,7 +258,7 @@ public class TestAggregatedLogFormat {
               .currentTimeMillis())).length() : 0)
             + ("\nLogLength:" + numChars).length()
             + "\nLog Contents:\n".length() + numChars + "\n".length()
-            + "End of LogType:stdout\n".length();
+            + "\nEnd of LogType:stdout\n".length();
     Assert.assertTrue("LogType not matched", s.contains("LogType:stdout"));
     Assert.assertTrue("log file:stderr should not be aggregated.", !s.contains("LogType:stderr"));
     Assert.assertTrue("log file:logs should not be aggregated.", !s.contains("LogType:logs"));

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c35fa4a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java
index 692b172..b56fae9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java
@@ -404,20 +404,19 @@ public class AHSWebServices extends WebServices {
 
                     long toSkip = 0;
                     long totalBytesToRead = fileLength;
+                    long skipAfterRead = 0;
                     if (bytes < 0) {
                       long absBytes = Math.abs(bytes);
                       if (absBytes < fileLength) {
                         toSkip = fileLength - absBytes;
                         totalBytesToRead = absBytes;
                       }
-                      long skippedBytes = valueStream.skip(toSkip);
-                      if (skippedBytes != toSkip) {
-                        throw new IOException("The bytes were skipped are "
-                            + "different from the caller requested");
-                      }
+                      org.apache.hadoop.io.IOUtils.skipFully(
+                          valueStream, toSkip);
                     } else {
                       if (bytes < fileLength) {
                         totalBytesToRead = bytes;
+                        skipAfterRead = fileLength - bytes;
                       }
                     }
 
@@ -435,6 +434,8 @@ public class AHSWebServices extends WebServices {
                           : (int) pendingRead;
                       len = valueStream.read(buf, 0, toRead);
                     }
+                    org.apache.hadoop.io.IOUtils.skipFully(
+                        valueStream, skipAfterRead);
                     sb = new StringBuilder();
                     sb.append("\nEnd of LogType:" + fileType + "\n");
                     b = sb.toString().getBytes(Charset.forName("UTF-8"));

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c35fa4a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
index 3a30392..4e38e90 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
@@ -264,20 +264,18 @@ public class NMWebServices {
             byte[] buf = new byte[bufferSize];
             long toSkip = 0;
             long totalBytesToRead = fileLength;
+            long skipAfterRead = 0;
             if (bytes < 0) {
               long absBytes = Math.abs(bytes);
               if (absBytes < fileLength) {
                 toSkip = fileLength - absBytes;
                 totalBytesToRead = absBytes;
               }
-              long skippedBytes = fis.skip(toSkip);
-              if (skippedBytes != toSkip) {
-                throw new IOException("The bytes were skipped are different "
-                    + "from the caller requested");
-              }
+              org.apache.hadoop.io.IOUtils.skipFully(fis, toSkip);
             } else {
               if (bytes < fileLength) {
                 totalBytesToRead = bytes;
+                skipAfterRead = fileLength - bytes;
               }
             }
 
@@ -295,6 +293,7 @@ public class NMWebServices {
                   : (int) pendingRead;
               len = fis.read(buf, 0, toRead);
             }
+            org.apache.hadoop.io.IOUtils.skipFully(fis, skipAfterRead);
             os.flush();
           } finally {
             IOUtils.closeQuietly(fis);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c35fa4a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
index c98e366..1edb841 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
@@ -948,7 +948,7 @@ public class TestLogAggregationService extends BaseContainerManagerTest
{
         Assert.assertEquals(numOfLogsPerContainer, thisContainerMap.size());
         for (String fileType : logFiles) {
           String expectedValue =
-              containerStr + " Hello " + fileType + "!End of LogType:"
+              containerStr + " Hello " + fileType + "!\nEnd of LogType:"
                   + fileType;
           LOG.info("Expected log-content : " + new String(expectedValue));
           String foundValue = thisContainerMap.remove(fileType);


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message