hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From junping...@apache.org
Subject hadoop git commit: YARN-7037. Optimize data transfer with zero-copy approach for containerlogs REST API in NMWebServices. Contributed by Tao Yang.
Date Tue, 29 Aug 2017 22:14:23 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk cc8893edc -> ad45d1999


YARN-7037. Optimize data transfer with zero-copy approach for containerlogs REST API in NMWebServices.
Contributed by Tao Yang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ad45d199
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ad45d199
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ad45d199

Branch: refs/heads/trunk
Commit: ad45d19998c1b0da25754d0016854046731fa623
Parents: cc8893e
Author: Junping Du <junping_du@apache.org>
Authored: Tue Aug 29 15:16:23 2017 -0700
Committer: Junping Du <junping_du@apache.org>
Committed: Tue Aug 29 15:16:23 2017 -0700

----------------------------------------------------------------------
 .../yarn/logaggregation/LogToolUtils.java       | 52 ++++++++++++++++++++
 .../nodemanager/webapp/NMWebServices.java       | 10 ++--
 2 files changed, 57 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad45d199/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogToolUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogToolUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogToolUtils.java
index ae2517a..74f694e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogToolUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogToolUtils.java
@@ -19,9 +19,13 @@ package org.apache.hadoop.yarn.logaggregation;
 
 import java.io.DataInputStream;
 import java.io.EOFException;
+import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.nio.channels.Channels;
+import java.nio.channels.FileChannel;
+import java.nio.channels.WritableByteChannel;
 import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.List;
@@ -195,6 +199,54 @@ public final class LogToolUtils {
     os.flush();
   }
 
+  public static void outputContainerLogThroughZeroCopy(String containerId,
+      String nodeId, String fileName, long fileLength, long outputSize,
+      String lastModifiedTime, FileInputStream fis, OutputStream os,
+      ContainerLogAggregationType logType) throws IOException {
+    long toSkip = 0;
+    long totalBytesToRead = fileLength;
+    if (outputSize < 0) {
+      long absBytes = Math.abs(outputSize);
+      if (absBytes < fileLength) {
+        toSkip = fileLength - absBytes;
+        totalBytesToRead = absBytes;
+      }
+    } else {
+      if (outputSize < fileLength) {
+        totalBytesToRead = outputSize;
+      }
+    }
+
+    if (totalBytesToRead > 0) {
+      // output log summary
+      StringBuilder sb = new StringBuilder();
+      String containerStr = String.format(
+          LogToolUtils.CONTAINER_ON_NODE_PATTERN,
+          containerId, nodeId);
+      sb.append(containerStr + "\n");
+      sb.append("LogAggregationType: " + logType + "\n");
+      sb.append(StringUtils.repeat("=", containerStr.length()) + "\n");
+      sb.append("LogType:" + fileName + "\n");
+      sb.append("LogLastModifiedTime:" + lastModifiedTime + "\n");
+      sb.append("LogLength:" + Long.toString(fileLength) + "\n");
+      sb.append("LogContents:\n");
+      byte[] b = sb.toString().getBytes(
+          Charset.forName("UTF-8"));
+      os.write(b, 0, b.length);
+      // output log content
+      FileChannel inputChannel = fis.getChannel();
+      WritableByteChannel outputChannel = Channels.newChannel(os);
+      long position = toSkip;
+      while (totalBytesToRead > 0) {
+        long transferred =
+            inputChannel.transferTo(position, totalBytesToRead, outputChannel);
+        totalBytesToRead -= transferred;
+        position += transferred;
+      }
+      os.flush();
+    }
+  }
+
   public static boolean outputAggregatedContainerLog(Configuration conf,
       ApplicationId appId, String appOwner,
       String containerId, String nodeId,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad45d199/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
index 7a6aa0f..3ba7d5c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
@@ -426,11 +426,9 @@ public class NMWebServices {
         public void write(OutputStream os) throws IOException,
             WebApplicationException {
           try {
-            int bufferSize = 65536;
-            byte[] buf = new byte[bufferSize];
-            LogToolUtils.outputContainerLog(containerId.toString(),
-                nmContext.getNodeId().toString(), outputFileName, fileLength,
-                bytes, lastModifiedTime, fis, os, buf,
+            LogToolUtils.outputContainerLogThroughZeroCopy(
+                containerId.toString(), nmContext.getNodeId().toString(),
+                outputFileName, fileLength, bytes, lastModifiedTime, fis, os,
                 ContainerLogAggregationType.LOCAL);
             StringBuilder sb = new StringBuilder();
             String endOfFile = "End of LogType:" + outputFileName;
@@ -451,6 +449,8 @@ public class NMWebServices {
             Application app = nmContext.getApplications().get(appId);
             String appOwner = app == null ? null : app.getUser();
             try {
+              int bufferSize = 65536;
+              byte[] buf = new byte[bufferSize];
               LogToolUtils.outputAggregatedContainerLog(nmContext.getConf(),
                   appId, appOwner, containerId.toString(),
                   nmContext.getNodeId().toString(), outputFileName, bytes,


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message