hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zjs...@apache.org
Subject [19/50] [abbrv] hadoop git commit: HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than DFSOutputStream#writeChunk (cmccabe)
Date Mon, 06 Apr 2015 19:22:49 GMT
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than DFSOutputStream#writeChunk
(cmccabe)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f4afce08
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f4afce08
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f4afce08

Branch: refs/heads/YARN-2928
Commit: f4afce0801bc488e301aee820a8091252bbc2748
Parents: f066e73
Author: Colin Patrick Mccabe <cmccabe@cloudera.com>
Authored: Wed Apr 1 13:55:40 2015 -0700
Committer: Zhijie Shen <zjshen@apache.org>
Committed: Mon Apr 6 12:08:12 2015 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/fs/FSOutputSummer.java    | 20 ++++++++++++++++----
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt     |  3 +++
 .../org/apache/hadoop/hdfs/DFSOutputStream.java | 15 ++++-----------
 .../org/apache/hadoop/tracing/TestTracing.java  |  4 ++--
 4 files changed, 25 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4afce08/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java
index 13a5e26..d2998b6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.fs;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.util.DataChecksum;
+import org.apache.htrace.NullScope;
+import org.apache.htrace.TraceScope;
 
 import java.io.IOException;
 import java.io.OutputStream;
@@ -194,16 +196,26 @@ abstract public class FSOutputSummer extends OutputStream {
     return sum.getChecksumSize();
   }
 
+  protected TraceScope createWriteTraceScope() {
+    return NullScope.INSTANCE;
+  }
+
   /** Generate checksums for the given data chunks and output chunks & checksums
    * to the underlying output stream.
    */
   private void writeChecksumChunks(byte b[], int off, int len)
   throws IOException {
     sum.calculateChunkedSums(b, off, len, checksum, 0);
-    for (int i = 0; i < len; i += sum.getBytesPerChecksum()) {
-      int chunkLen = Math.min(sum.getBytesPerChecksum(), len - i);
-      int ckOffset = i / sum.getBytesPerChecksum() * getChecksumSize();
-      writeChunk(b, off + i, chunkLen, checksum, ckOffset, getChecksumSize());
+    TraceScope scope = createWriteTraceScope();
+    try {
+      for (int i = 0; i < len; i += sum.getBytesPerChecksum()) {
+        int chunkLen = Math.min(sum.getBytesPerChecksum(), len - i);
+        int ckOffset = i / sum.getBytesPerChecksum() * getChecksumSize();
+        writeChunk(b, off + i, chunkLen, checksum, ckOffset,
+            getChecksumSize());
+      }
+    } finally {
+      scope.close();
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4afce08/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 435fdd7..b5591e0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -378,6 +378,9 @@ Release 2.8.0 - UNRELEASED
 
   OPTIMIZATIONS
 
+    HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
+    DFSOutputStream#writeChunk (cmccabe)
+
   BUG FIXES
 
     HDFS-7501. TransactionsSinceLastCheckpoint can be negative on SBNs.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4afce08/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
index 933d8e6..c88639d 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
@@ -372,21 +372,14 @@ public class DFSOutputStream extends FSOutputSummer
     }
   }
 
+  protected TraceScope createWriteTraceScope() {
+    return dfsClient.getPathTraceScope("DFSOutputStream#write", src);
+  }
+
   // @see FSOutputSummer#writeChunk()
   @Override
   protected synchronized void writeChunk(byte[] b, int offset, int len,
       byte[] checksum, int ckoff, int cklen) throws IOException {
-    TraceScope scope =
-        dfsClient.getPathTraceScope("DFSOutputStream#writeChunk", src);
-    try {
-      writeChunkImpl(b, offset, len, checksum, ckoff, cklen);
-    } finally {
-      scope.close();
-    }
-  }
-
-  private synchronized void writeChunkImpl(byte[] b, int offset, int len,
-          byte[] checksum, int ckoff, int cklen) throws IOException {
     dfsClient.checkOpen();
     checkClosed();
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4afce08/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracing.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracing.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracing.java
index 3720abe..01361b5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracing.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracing.java
@@ -89,7 +89,7 @@ public class TestTracing {
       "org.apache.hadoop.hdfs.protocol.ClientProtocol.complete",
       "ClientNamenodeProtocol#complete",
       "newStreamForCreate",
-      "DFSOutputStream#writeChunk",
+      "DFSOutputStream#write",
       "DFSOutputStream#close",
       "dataStreamer",
       "OpWriteBlockProto",
@@ -117,7 +117,7 @@ public class TestTracing {
       "org.apache.hadoop.hdfs.protocol.ClientProtocol.complete",
       "ClientNamenodeProtocol#complete",
       "newStreamForCreate",
-      "DFSOutputStream#writeChunk",
+      "DFSOutputStream#write",
       "DFSOutputStream#close",
     };
     for (String desc : spansInTopTrace) {


Mime
View raw message