hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From prasan...@apache.org
Subject hive git commit: HIVE-11945: ORC with non-local reads may not be reusing connection to DN (Rajesh Balamohan reviewed by Sergey Shelukhin, Prasanth Jayachandran)
Date Tue, 29 Sep 2015 01:11:03 GMT
Repository: hive
Updated Branches:
  refs/heads/master 392145812 -> a4c43f033


HIVE-11945: ORC with non-local reads may not be reusing connection to DN (Rajesh Balamohan
reviewed by Sergey Shelukhin, Prasanth Jayachandran)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a4c43f03
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a4c43f03
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a4c43f03

Branch: refs/heads/master
Commit: a4c43f0335b33a75d2e9f3dc53b3cd33f8f115cf
Parents: 3921458
Author: Prasanth Jayachandran <j.prasanth.j@gmail.com>
Authored: Mon Sep 28 20:10:50 2015 -0500
Committer: Prasanth Jayachandran <j.prasanth.j@gmail.com>
Committed: Mon Sep 28 20:10:50 2015 -0500

----------------------------------------------------------------------
 .../apache/hadoop/hive/ql/io/orc/MetadataReader.java    |  6 ++----
 .../org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java    | 12 +++++-------
 .../apache/hadoop/hive/ql/io/orc/RecordReaderUtils.java |  5 +++--
 3 files changed, 10 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/a4c43f03/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java
index cdc0372..43d2933 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java
@@ -82,8 +82,7 @@ public class MetadataReader {
         }
         if ((included == null || included[col]) && indexes[col] == null) {
           byte[] buffer = new byte[len];
-          file.seek(offset);
-          file.readFully(buffer);
+          file.readFully(offset, buffer, 0, buffer.length);
           ByteBuffer[] bb = new ByteBuffer[] {ByteBuffer.wrap(buffer)};
           indexes[col] = OrcProto.RowIndex.parseFrom(InStream.create("index",
               bb, new long[]{0}, stream.getLength(), codec, bufferSize));
@@ -108,8 +107,7 @@ public class MetadataReader {
 
     // read the footer
     ByteBuffer tailBuf = ByteBuffer.allocate(tailLength);
-    file.seek(offset);
-    file.readFully(tailBuf.array(), tailBuf.arrayOffset(), tailLength);
+    file.readFully(offset, tailBuf.array(), tailBuf.arrayOffset(), tailLength);
     return OrcProto.StripeFooter.parseFrom(InStream.create("footer",
         Lists.<DiskRange>newArrayList(new BufferChunk(tailBuf, 0)),
         tailLength, codec, bufferSize));

http://git-wip-us.apache.org/repos/asf/hive/blob/a4c43f03/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
index ab539c4..23b3b55 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
@@ -244,9 +244,8 @@ public class ReaderImpl implements Reader {
     if (!Text.decode(array, offset, len).equals(OrcFile.MAGIC)) {
       // If it isn't there, this may be the 0.11.0 version of ORC.
       // Read the first 3 bytes of the file to check for the header
-      in.seek(0);
       byte[] header = new byte[len];
-      in.readFully(header, 0, len);
+      in.readFully(0, header, 0, len);
       // if it isn't there, this isn't an ORC file
       if (!Text.decode(header, 0 , len).equals(OrcFile.MAGIC)) {
         throw new FileFormatException("Malformed ORC file " + path +
@@ -472,10 +471,10 @@ public class ReaderImpl implements Reader {
 
     //read last bytes into buffer to get PostScript
     int readSize = (int) Math.min(size, DIRECTORY_SIZE_GUESS);
-    file.seek(size - readSize);
     ByteBuffer buffer = ByteBuffer.allocate(readSize);
     assert buffer.position() == 0;
-    file.readFully(buffer.array(), buffer.arrayOffset(), readSize);
+    file.readFully((size - readSize),
+        buffer.array(), buffer.arrayOffset(), readSize);
     buffer.position(0);
 
     //read the PostScript
@@ -495,10 +494,9 @@ public class ReaderImpl implements Reader {
     int extra = Math.max(0, psLen + 1 + footerSize + metadataSize - readSize);
     if (extra > 0) {
       //more bytes need to be read, seek back to the right place and read extra bytes
-      file.seek(size - readSize - extra);
       ByteBuffer extraBuf = ByteBuffer.allocate(extra + readSize);
-      file.readFully(extraBuf.array(),
-        extraBuf.arrayOffset() + extraBuf.position(), extra);
+      file.readFully((size - readSize - extra), extraBuf.array(),
+          extraBuf.arrayOffset() + extraBuf.position(), extra);
       extraBuf.position(extra);
       //append with already read bytes
       extraBuf.put(buffer);

http://git-wip-us.apache.org/repos/asf/hive/blob/a4c43f03/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderUtils.java
index ded3979..9c9a1c0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderUtils.java
@@ -246,8 +246,8 @@ public class RecordReaderUtils {
       }
       int len = (int) (range.getEnd() - range.getOffset());
       long off = range.getOffset();
-      file.seek(base + off);
       if (zcr != null) {
+        file.seek(base + off);
         boolean hasReplaced = false;
         while (len > 0) {
           ByteBuffer partial = zcr.readBuffer(len, false);
@@ -264,12 +264,13 @@ public class RecordReaderUtils {
           off += read;
         }
       } else if (doForceDirect) {
+        file.seek(base + off);
         ByteBuffer directBuf = ByteBuffer.allocateDirect(len);
         readDirect(file, len, directBuf);
         range = range.replaceSelfWith(new BufferChunk(directBuf, range.getOffset()));
       } else {
         byte[] buffer = new byte[len];
-        file.readFully(buffer, 0, buffer.length);
+        file.readFully((base + off), buffer, 0, buffer.length);
         range = range.replaceSelfWith(new BufferChunk(ByteBuffer.wrap(buffer), range.getOffset()));
       }
       range = range.next;


Mime
View raw message