hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e..@apache.org
Subject svn commit: r1037903 - in /hadoop/common/branches/branch-0.22: ./ CHANGES.txt src/contrib/ec2/ src/docs/ src/java/ src/java/org/apache/hadoop/io/compress/zlib/ZlibCompressor.java src/test/core/ src/test/core/org/apache/hadoop/io/TestSequenceFile.java
Date Mon, 22 Nov 2010 21:54:58 GMT
Author: eli
Date: Mon Nov 22 21:54:58 2010
New Revision: 1037903

URL: http://svn.apache.org/viewvc?rev=1037903&view=rev
Log:
HADOOP-6683. svn merge -c 1037901 from trunk

Modified:
    hadoop/common/branches/branch-0.22/   (props changed)
    hadoop/common/branches/branch-0.22/CHANGES.txt   (contents, props changed)
    hadoop/common/branches/branch-0.22/src/contrib/ec2/   (props changed)
    hadoop/common/branches/branch-0.22/src/docs/   (props changed)
    hadoop/common/branches/branch-0.22/src/java/   (props changed)
    hadoop/common/branches/branch-0.22/src/java/org/apache/hadoop/io/compress/zlib/ZlibCompressor.java
    hadoop/common/branches/branch-0.22/src/test/core/   (props changed)
    hadoop/common/branches/branch-0.22/src/test/core/org/apache/hadoop/io/TestSequenceFile.java
  (props changed)

Propchange: hadoop/common/branches/branch-0.22/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Nov 22 21:54:58 2010
@@ -1 +1,2 @@
+/hadoop/common/trunk:1037901
 /hadoop/core/branches/branch-0.19/core:713112

Modified: hadoop/common/branches/branch-0.22/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/CHANGES.txt?rev=1037903&r1=1037902&r2=1037903&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.22/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.22/CHANGES.txt Mon Nov 22 21:54:58 2010
@@ -190,6 +190,9 @@ Release 0.22.0 - Unreleased
     HADOOP-6884. Add LOG.isDebugEnabled() guard for each LOG.debug(..).
     (Erik Steffl via szetszwo)
 
+    HADOOP-6683. ZlibCompressor does not fully utilize the buffer.
+    (Kang Xiao via eli)
+
   BUG FIXES
 
     HADOOP-6638. try to relogin in a case of failed RPC connection (expired 

Propchange: hadoop/common/branches/branch-0.22/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Nov 22 21:54:58 2010
@@ -1,3 +1,4 @@
+/hadoop/common/trunk/CHANGES.txt:1037901
 /hadoop/core/branches/branch-0.18/CHANGES.txt:727226
 /hadoop/core/branches/branch-0.19/CHANGES.txt:713112
 /hadoop/core/trunk/CHANGES.txt:776175-785643,785929-786278

Propchange: hadoop/common/branches/branch-0.22/src/contrib/ec2/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Nov 22 21:54:58 2010
@@ -1,2 +1,3 @@
+/hadoop/common/trunk/src/contrib/ec2:1037901
 /hadoop/core/branches/branch-0.19/core/src/contrib/ec2:713112
 /hadoop/core/trunk/src/contrib/ec2:776175-784663

Propchange: hadoop/common/branches/branch-0.22/src/docs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Nov 22 21:54:58 2010
@@ -1 +1,2 @@
+/hadoop/common/trunk/src/docs:1037901
 /hadoop/core/branches/branch-0.19/src/docs:713112

Propchange: hadoop/common/branches/branch-0.22/src/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Nov 22 21:54:58 2010
@@ -1,2 +1,3 @@
+/hadoop/common/trunk/src/java:1037901
 /hadoop/core/branches/branch-0.19/core/src/java:713112
 /hadoop/core/trunk/src/core:776175-785643,785929-786278

Modified: hadoop/common/branches/branch-0.22/src/java/org/apache/hadoop/io/compress/zlib/ZlibCompressor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/src/java/org/apache/hadoop/io/compress/zlib/ZlibCompressor.java?rev=1037903&r1=1037902&r2=1037903&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.22/src/java/org/apache/hadoop/io/compress/zlib/ZlibCompressor.java
(original)
+++ hadoop/common/branches/branch-0.22/src/java/org/apache/hadoop/io/compress/zlib/ZlibCompressor.java
Mon Nov 22 21:54:58 2010
@@ -53,6 +53,7 @@ public class ZlibCompressor implements C
   private int userBufOff = 0, userBufLen = 0;
   private Buffer uncompressedDirectBuf = null;
   private int uncompressedDirectBufOff = 0, uncompressedDirectBufLen = 0;
+  private boolean keepUncompressedBuf = false;
   private Buffer compressedDirectBuf = null;
   private boolean finish, finished;
 
@@ -269,6 +270,7 @@ public class ZlibCompressor implements C
     this.userBuf = b;
     this.userBufOff = off;
     this.userBufLen = len;
+    uncompressedDirectBufOff = 0;
     setInputFromSavedData();
     
     // Reinitialize zlib's output direct buffer 
@@ -276,21 +278,13 @@ public class ZlibCompressor implements C
     compressedDirectBuf.position(directBufferSize);
   }
   
+  //copy enough data from userBuf to uncompressedDirectBuf
   synchronized void setInputFromSavedData() {
-    uncompressedDirectBufOff = 0;
-    uncompressedDirectBufLen = userBufLen;
-    if (uncompressedDirectBufLen > directBufferSize) {
-      uncompressedDirectBufLen = directBufferSize;
-    }
-
-    // Reinitialize zlib's input direct buffer
-    uncompressedDirectBuf.rewind();
-    ((ByteBuffer)uncompressedDirectBuf).put(userBuf, userBufOff,  
-                                            uncompressedDirectBufLen);
-
-    // Note how much data is being fed to zlib
-    userBufOff += uncompressedDirectBufLen;
-    userBufLen -= uncompressedDirectBufLen;
+    int len = Math.min(userBufLen, uncompressedDirectBuf.remaining());
+    ((ByteBuffer)uncompressedDirectBuf).put(userBuf, userBufOff, len);
+    userBufLen -= len;
+    userBufOff += len;
+    uncompressedDirectBufLen = uncompressedDirectBuf.position();
   }
 
   public synchronized void setDictionary(byte[] b, int off, int len) {
@@ -310,12 +304,21 @@ public class ZlibCompressor implements C
     }
 
     // Check if zlib has consumed all input
-    if (uncompressedDirectBufLen <= 0) {
+    // compress should be invoked if keepUncompressedBuf true
+    if (keepUncompressedBuf && uncompressedDirectBufLen > 0)
+      return false;
+    
+    if (uncompressedDirectBuf.remaining() > 0) {
       // Check if we have consumed all user-input
       if (userBufLen <= 0) {
         return true;
       } else {
+        // copy enough data from userBuf to uncompressedDirectBuf
         setInputFromSavedData();
+        if (uncompressedDirectBuf.remaining() > 0) // uncompressedDirectBuf is not full
+          return true;
+        else 
+          return false;
       }
     }
     
@@ -359,6 +362,17 @@ public class ZlibCompressor implements C
     n = deflateBytesDirect();
     compressedDirectBuf.limit(n);
     
+    // Check if zlib consumed all input buffer
+    // set keepUncompressedBuf properly
+    if (uncompressedDirectBufLen <= 0) { // zlib consumed all input buffer
+      keepUncompressedBuf = false;
+      uncompressedDirectBuf.clear();
+      uncompressedDirectBufOff = 0;
+      uncompressedDirectBufLen = 0;
+    } else { // zlib did not consume all input buffer
+      keepUncompressedBuf = true;
+    }
+    
     // Get atmost 'len' bytes
     n = Math.min(n, len);
     ((ByteBuffer)compressedDirectBuf).get(b, off, n);
@@ -393,6 +407,7 @@ public class ZlibCompressor implements C
     finished = false;
     uncompressedDirectBuf.rewind();
     uncompressedDirectBufOff = uncompressedDirectBufLen = 0;
+    keepUncompressedBuf = false;
     compressedDirectBuf.limit(directBufferSize);
     compressedDirectBuf.position(directBufferSize);
     userBufOff = userBufLen = 0;

Propchange: hadoop/common/branches/branch-0.22/src/test/core/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Nov 22 21:54:58 2010
@@ -1,2 +1,3 @@
+/hadoop/common/trunk/src/test/core:1037901
 /hadoop/core/branches/branch-0.19/core/src/test/core:713112
 /hadoop/core/trunk/src/test/core:776175-785643,785929-786278

Propchange: hadoop/common/branches/branch-0.22/src/test/core/org/apache/hadoop/io/TestSequenceFile.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Nov 22 21:54:58 2010
@@ -1,2 +1,3 @@
+/hadoop/common/trunk/src/test/core/org/apache/hadoop/io/TestSequenceFile.java:1037901
 /hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/io/TestSequenceFile.java:713112
 /hadoop/core/trunk/src/test/mapred/org/apache/hadoop/io/TestSequenceFile.java:776175-785643



Mime
View raw message