hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gangtim...@apache.org
Subject svn commit: r1462363 - /hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OutStream.java
Date Fri, 29 Mar 2013 00:26:47 GMT
Author: gangtimliu
Date: Fri Mar 29 00:26:46 2013
New Revision: 1462363

URL: http://svn.apache.org/r1462363
Log:
HIVE-4157: ORC runs out of heap when writing (Kevin Wilfong vi Gang Tim Liu)

Modified:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OutStream.java

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OutStream.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OutStream.java?rev=1462363&r1=1462362&r2=1462363&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OutStream.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OutStream.java Fri Mar 29 00:26:46
2013
@@ -45,17 +45,14 @@ class OutStream extends PositionedOutput
     this.bufferSize = bufferSize;
     this.codec = codec;
     this.receiver = receiver;
-    getNewInputBuffer();
   }
 
   public void clear() throws IOException {
-    current.position(codec == null ? 0 : HEADER_SIZE);
-    if (compressed != null) {
-      compressed.clear();
-    }
-    if (overflow != null) {
-      overflow.clear();
-    }
+    uncompressedBytes = 0;
+    compressedBytes = 0;
+    compressed = null;
+    overflow = null;
+    current = null;
   }
 
   /**
@@ -100,6 +97,9 @@ class OutStream extends PositionedOutput
 
   @Override
   public void write(int i) throws IOException {
+    if (current == null) {
+      getNewInputBuffer();
+    }
     if (current.remaining() < 1) {
       spill();
     }
@@ -109,6 +109,9 @@ class OutStream extends PositionedOutput
 
   @Override
   public void write(byte[] bytes, int offset, int length) throws IOException {
+    if (current == null) {
+      getNewInputBuffer();
+    }
     int remaining = Math.min(current.remaining(), length);
     current.put(bytes, offset, remaining);
     uncompressedBytes += remaining;
@@ -125,7 +128,7 @@ class OutStream extends PositionedOutput
 
   private void spill() throws java.io.IOException {
     // if there isn't anything in the current buffer, don't spill
-    if (current.position() == (codec == null ? 0 : HEADER_SIZE)) {
+    if (current == null || current.position() == (codec == null ? 0 : HEADER_SIZE)) {
       return;
     }
     flip();
@@ -211,8 +214,7 @@ class OutStream extends PositionedOutput
       receiver.output(compressed);
       compressed = null;
     }
-    uncompressedBytes = 0;
-    compressedBytes = 0;
+    clear();
   }
 
   @Override



Mime
View raw message