hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cmcc...@apache.org
Subject hadoop git commit: HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than DFSOutputStream#writeChunk (cmccabe)
Date Wed, 01 Apr 2015 21:10:47 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 6d0bebc91 -> 4b74aa718


HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than DFSOutputStream#writeChunk
(cmccabe)

(cherry picked from commit c94d594a57806dec515e2a2053a1221f8ce48cc4)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4b74aa71
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4b74aa71
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4b74aa71

Branch: refs/heads/branch-2
Commit: 4b74aa7182c44d61a339f43c691dc31d14028ca2
Parents: 6d0bebc
Author: Colin Patrick Mccabe <cmccabe@cloudera.com>
Authored: Wed Apr 1 13:55:40 2015 -0700
Committer: Colin Patrick Mccabe <cmccabe@cloudera.com>
Committed: Wed Apr 1 14:10:25 2015 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/fs/FSOutputSummer.java    | 20 ++++++++++++++++----
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt     |  3 +++
 .../org/apache/hadoop/hdfs/DFSOutputStream.java | 15 ++++-----------
 .../org/apache/hadoop/tracing/TestTracing.java  |  4 ++--
 4 files changed, 25 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b74aa71/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java
index 13a5e26..d2998b6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.fs;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.util.DataChecksum;
+import org.apache.htrace.NullScope;
+import org.apache.htrace.TraceScope;
 
 import java.io.IOException;
 import java.io.OutputStream;
@@ -194,16 +196,26 @@ abstract public class FSOutputSummer extends OutputStream {
     return sum.getChecksumSize();
   }
 
+  protected TraceScope createWriteTraceScope() {
+    return NullScope.INSTANCE;
+  }
+
   /** Generate checksums for the given data chunks and output chunks & checksums
    * to the underlying output stream.
    */
   private void writeChecksumChunks(byte b[], int off, int len)
   throws IOException {
     sum.calculateChunkedSums(b, off, len, checksum, 0);
-    for (int i = 0; i < len; i += sum.getBytesPerChecksum()) {
-      int chunkLen = Math.min(sum.getBytesPerChecksum(), len - i);
-      int ckOffset = i / sum.getBytesPerChecksum() * getChecksumSize();
-      writeChunk(b, off + i, chunkLen, checksum, ckOffset, getChecksumSize());
+    TraceScope scope = createWriteTraceScope();
+    try {
+      for (int i = 0; i < len; i += sum.getBytesPerChecksum()) {
+        int chunkLen = Math.min(sum.getBytesPerChecksum(), len - i);
+        int ckOffset = i / sum.getBytesPerChecksum() * getChecksumSize();
+        writeChunk(b, off + i, chunkLen, checksum, ckOffset,
+            getChecksumSize());
+      }
+    } finally {
+      scope.close();
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b74aa71/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index ca1a4e1..7f1640e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -63,6 +63,9 @@ Release 2.8.0 - UNRELEASED
 
   OPTIMIZATIONS
 
+    HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
+    DFSOutputStream#writeChunk (cmccabe)
+
   BUG FIXES
 
     HDFS-7501. TransactionsSinceLastCheckpoint can be negative on SBNs.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b74aa71/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
index 3aec8ac..0441f2f 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
@@ -372,21 +372,14 @@ public class DFSOutputStream extends FSOutputSummer
     }
   }
 
+  protected TraceScope createWriteTraceScope() {
+    return dfsClient.getPathTraceScope("DFSOutputStream#write", src);
+  }
+
   // @see FSOutputSummer#writeChunk()
   @Override
   protected synchronized void writeChunk(byte[] b, int offset, int len,
       byte[] checksum, int ckoff, int cklen) throws IOException {
-    TraceScope scope =
-        dfsClient.getPathTraceScope("DFSOutputStream#writeChunk", src);
-    try {
-      writeChunkImpl(b, offset, len, checksum, ckoff, cklen);
-    } finally {
-      scope.close();
-    }
-  }
-
-  private synchronized void writeChunkImpl(byte[] b, int offset, int len,
-          byte[] checksum, int ckoff, int cklen) throws IOException {
     dfsClient.checkOpen();
     checkClosed();
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b74aa71/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracing.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracing.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracing.java
index 3720abe..01361b5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracing.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracing.java
@@ -89,7 +89,7 @@ public class TestTracing {
       "org.apache.hadoop.hdfs.protocol.ClientProtocol.complete",
       "ClientNamenodeProtocol#complete",
       "newStreamForCreate",
-      "DFSOutputStream#writeChunk",
+      "DFSOutputStream#write",
       "DFSOutputStream#close",
       "dataStreamer",
       "OpWriteBlockProto",
@@ -117,7 +117,7 @@ public class TestTracing {
       "org.apache.hadoop.hdfs.protocol.ClientProtocol.complete",
       "ClientNamenodeProtocol#complete",
       "newStreamForCreate",
-      "DFSOutputStream#writeChunk",
+      "DFSOutputStream#write",
       "DFSOutputStream#close",
     };
     for (String desc : spansInTopTrace) {


Mime
View raw message