hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ni...@apache.org
Subject svn commit: r618349 - in /hadoop/core/trunk: CHANGES.txt src/java/org/apache/hadoop/dfs/DataNode.java
Date Mon, 04 Feb 2008 17:57:57 GMT
Author: nigel
Date: Mon Feb  4 09:57:54 2008
New Revision: 618349

URL: http://svn.apache.org/viewvc?rev=618349&view=rev
Log:
HADOOP-2768. Fix performance regression caused by HADOOP-1707. Contributed by dhruba borthakur.

Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DataNode.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=618349&r1=618348&r2=618349&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Mon Feb  4 09:57:54 2008
@@ -16,7 +16,7 @@
     HADOOP-2738 Text is not subclassable because set(Text) and compareTo(Object)
     access the other instance's private members directly.
 
-Release 0.16.0 - 2008-02-04
+Release 0.16.0 - 2008-02-07
 
   INCOMPATIBLE CHANGES
 
@@ -660,6 +660,9 @@
     
     HADOOP-2755. Fix fsck performance degradation because of permissions 
     issue.  (Tsz Wo (Nicholas), SZE via dhruba)
+
+    HADOOP-2768. Fix performance regression caused by HADOOP-1707.
+    (dhruba borthakur via nigel)
 
 Release 0.15.3 - 2008-01-18
 

Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DataNode.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DataNode.java?rev=618349&r1=618348&r2=618349&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DataNode.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DataNode.java Mon Feb  4 09:57:54 2008
@@ -1092,7 +1092,9 @@
             mirrorSock.connect(mirrorTarget, timeoutValue);
             mirrorSock.setSoTimeout(timeoutValue);
             mirrorSock.setSendBufferSize(DEFAULT_DATA_SOCKET_SIZE);
-            mirrorOut = new DataOutputStream(mirrorSock.getOutputStream());
+            mirrorOut = new DataOutputStream(
+                          new BufferedOutputStream(mirrorSock.getOutputStream(),
+                                                   BUFFER_SIZE));
             mirrorIn = new DataInputStream(mirrorSock.getInputStream());
 
             // Write header: Copied from DFSClient.java!
@@ -1918,6 +1920,18 @@
     }
   }
 
+  // this class is a bufferoutputstream that exposes the number of
+  // bytes in the buffer.
+  static private class DFSBufferedOutputStream extends BufferedOutputStream {
+    DFSBufferedOutputStream(OutputStream out, int capacity) {
+      super(out, capacity);
+    }
+
+    int count() {
+      return count;
+    }
+  }
+
   /* A class that receives a block and wites to its own disk, meanwhile
    * may copies it to another site. If a throttler is provided,
    * streaming throttling is also supported. 
@@ -1929,6 +1943,7 @@
     private DataChecksum checksum; // from where chunks of a block can be read
     private DataOutputStream out = null; // to block file at local disk
     private DataOutputStream checksumOut = null; // to crc file at local disk
+    private DFSBufferedOutputStream bufStream = null;
     private int bytesPerChecksum;
     private int checksumSize;
     private byte buf[];
@@ -1968,8 +1983,11 @@
         streams = data.writeToBlock(block, isRecovery);
 		this.finalized = data.isValidBlock(block);
         if (streams != null) {
-          this.out = new DataOutputStream(streams.dataOut);
-          this.checksumOut = new DataOutputStream(streams.checksumOut);
+          this.bufStream = new DFSBufferedOutputStream(
+                                          streams.dataOut, BUFFER_SIZE);
+          this.out = new DataOutputStream(bufStream);
+          this.checksumOut = new DataOutputStream(new BufferedOutputStream(
+                                          streams.checksumOut, BUFFER_SIZE));
         }
       } catch(IOException ioe) {
         IOUtils.closeStream(this);
@@ -2290,7 +2308,8 @@
         }
         return;
       }
-      if (data.getChannelPosition(block, streams) == offsetInBlock) {
+      if (data.getChannelPosition(block, streams) + bufStream.count() == 
+                                                    offsetInBlock) {
         return;                   // nothing to do 
       }
       if (offsetInBlock % bytesPerChecksum != 0) {
@@ -2657,5 +2676,4 @@
       System.exit(-1);
     }
   }
-
 }



Mime
View raw message