tajo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jh...@apache.org
Subject git commit: TAJO-156: BufferUnderflowException occurs during processing the intermediate file. (jinho)
Date Wed, 04 Sep 2013 14:51:31 GMT
Updated Branches:
  refs/heads/master fb37ff33c -> b2dc84fe7


TAJO-156: BufferUnderflowException occurs during processing the intermediate file. (jinho)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tajo/commit/b2dc84fe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tajo/tree/b2dc84fe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tajo/diff/b2dc84fe

Branch: refs/heads/master
Commit: b2dc84fe7cae0b265f2ba11a7b2511d760ccf73f
Parents: fb37ff3
Author: jinossy <jinossy@gmail.com>
Authored: Wed Sep 4 23:50:08 2013 +0900
Committer: jinossy <jinossy@gmail.com>
Committed: Wed Sep 4 23:50:08 2013 +0900

----------------------------------------------------------------------
 CHANGES.txt                                      |  3 +++
 .../planner/physical/TestExternalSortExec.java   |  2 +-
 .../java/org/apache/tajo/storage/RawFile.java    | 19 ++++++++++++++++++-
 3 files changed, 22 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/b2dc84fe/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 3f87a23..c16c833 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -124,6 +124,9 @@ Release 0.2.0 - unreleased
 
   BUG FIXES
 
+    TAJO-156: BufferUnderflowException occurs during processing the 
+    intermediate file. (jinho)
+
     TAJO-151: Multiple union queries within a table subquery causes NPE.
     (hyunsik)
 

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/b2dc84fe/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
index e7344d4..621f5a2 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
@@ -166,7 +166,7 @@ public class TestExternalSortExec {
       cnt++;
     }
     assertEquals(numTuple, cnt);
-
+    exec.close();
     System.out.println("Sort Time: " + (end - start) + " msc");
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/b2dc84fe/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RawFile.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RawFile.java
b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RawFile.java
index 7c0ea87..5c980d9 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RawFile.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RawFile.java
@@ -18,6 +18,8 @@
 
 package org.apache.tajo.storage;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.tajo.catalog.TableMeta;
@@ -38,6 +40,8 @@ import java.nio.channels.FileChannel;
 import java.util.Arrays;
 
 public class RawFile {
+  private static final Log LOG = LogFactory.getLog(RawFile.class);
+
   public static class RawFileScanner extends FileScanner implements SeekableScanner {
     private FileChannel channel;
     private DataType[] columnTypes;
@@ -50,6 +54,8 @@ public class RawFile {
     private BitArray nullFlags;
     private static final int RECORD_SIZE = 4;
     private int numBitsOfNullFlags;
+    private boolean eof = false;
+    private long fileSize;
 
     public RawFileScanner(Configuration conf, TableMeta meta, Path path) throws IOException
{
       super(conf, meta, null);
@@ -68,8 +74,13 @@ public class RawFile {
       URI uri = path.toUri();
       RandomAccessFile raf = new RandomAccessFile(new File(uri), "r");
       channel = raf.getChannel();
+      fileSize = channel.size();
 
-      buffer = ByteBuffer.allocateDirect(65535);
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("RawFileScanner open:" + path + "," + channel.position() + ", size :" +
channel.size());
+      }
+
+      buffer = ByteBuffer.allocateDirect(65535 * 4);
 
       columnTypes = new DataType[schema.getColumnNum()];
       for (int i = 0; i < schema.getColumnNum(); i++) {
@@ -111,6 +122,7 @@ public class RawFile {
 
     @Override
     public Tuple next() throws IOException {
+      if(eof) return null;
 
       if (buffer.remaining() < headerSize) {
         if (!fillBuffer()) {
@@ -223,6 +235,9 @@ public class RawFile {
         }
       }
 
+      if(!buffer.hasRemaining() && channel.position() == fileSize){
+        eof = true;
+      }
       return tuple;
     }
 
@@ -234,10 +249,12 @@ public class RawFile {
       channel.position(0);
       channel.read(buffer);
       buffer.flip();
+      eof = false;
     }
 
     @Override
     public void close() throws IOException {
+      buffer.clear();
       channel.close();
     }
 


Mime
View raw message