hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject svn commit: r1662595 - in /hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc: EncodedReaderImpl.java InStream.java RecordReaderUtils.java
Date Thu, 26 Feb 2015 22:56:32 GMT
Author: sershe
Date: Thu Feb 26 22:56:32 2015
New Revision: 1662595

URL: http://svn.apache.org/r1662595
Log:
Fix issue w/0-length streams

Modified:
    hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/EncodedReaderImpl.java
    hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/InStream.java
    hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderUtils.java

Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/EncodedReaderImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/EncodedReaderImpl.java?rev=1662595&r1=1662594&r2=1662595&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/EncodedReaderImpl.java
(original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/EncodedReaderImpl.java
Thu Feb 26 22:56:32 2015
@@ -273,8 +273,12 @@ public class EncodedReaderImpl implement
               // it when building the last RG, so each RG processing will decref once, and
the
               // last one will unlock the buffers.
               sctx.stripeLevelStream.incRef();
-              iter = InStream.uncompressStream(fileName, stripeOffset, iter, sctx.offset,
-                  sctx.offset + sctx.length, zcr, codec, bufferSize, cache, sctx.stripeLevelStream);
+              DiskRangeList lastCached = InStream.uncompressStream(fileName, stripeOffset,
iter,
+                  sctx.offset,sctx.offset + sctx.length, zcr, codec, bufferSize, cache,
+                  sctx.stripeLevelStream);
+              if (lastCached != null) {
+                iter = lastCached;
+              }
             }
             if (!isLastRg) {
               sctx.stripeLevelStream.incRef();
@@ -296,9 +300,11 @@ public class EncodedReaderImpl implement
             }
             boolean isStartOfStream = sctx.bufferIter == null;
             DiskRangeList range = isStartOfStream ? iter : sctx.bufferIter;
-            DiskRangeList next = InStream.uncompressStream(fileName, stripeOffset, range,
cOffset,
-                endCOffset, zcr, codec, bufferSize, cache, cb);
-            sctx.bufferIter = iter = next; // Reset iter just to ensure it's valid
+            DiskRangeList lastCached = InStream.uncompressStream(fileName, stripeOffset,
range,
+                cOffset, endCOffset, zcr, codec, bufferSize, cache, cb);
+            if (lastCached != null) {
+              sctx.bufferIter = iter = lastCached; // Reset iter just to ensure it's valid
+            }
           }
           ecb.setStreamData(colIxMod, streamIx, cb);
         }

Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/InStream.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/InStream.java?rev=1662595&r1=1662594&r2=1662595&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/InStream.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/InStream.java Thu Feb
26 22:56:32 2015
@@ -565,9 +565,10 @@ public abstract class InStream extends I
    */
   public static DiskRangeList uncompressStream(String fileName, long baseOffset,
       DiskRangeList start, long cOffset, long endCOffset, ZeroCopyReaderShim zcr,
-      CompressionCodec codec,int bufferSize, LowLevelCache cache, StreamBuffer streamBuffer)
+      CompressionCodec codec, int bufferSize, LowLevelCache cache, StreamBuffer streamBuffer)
           throws IOException {
     streamBuffer.cacheBuffers = new ArrayList<LlapMemoryBuffer>();
+    if (cOffset == endCOffset) return null;
     List<ProcCacheChunk> toDecompress = null;
     List<ByteBuffer> toRelease = null;
 

Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderUtils.java?rev=1662595&r1=1662594&r2=1662595&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderUtils.java
(original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderUtils.java
Thu Feb 26 22:56:32 2015
@@ -302,6 +302,7 @@ public class RecordReaderUtils {
   static List<DiskRange> getStreamBuffers(DiskRangeList range, long offset, long length)
{
     // This assumes sorted ranges (as do many other parts of ORC code.
     ArrayList<DiskRange> buffers = new ArrayList<DiskRange>();
+    if (length == 0) return buffers;
     long streamEnd = offset + length;
     boolean inRange = false;
     while (range != null) {



Mime
View raw message