hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject [1/8] hive git commit: HIVE-16547 : LLAP: may not unlock buffers in some cases (Sergey Shelukhin, reviewed by Prasanth Jayachandran)
Date Thu, 27 Apr 2017 20:07:49 GMT
Repository: hive
Updated Branches:
  refs/heads/branch-2 96784f7ca -> 4025eddd4
  refs/heads/branch-2.2 9bedccc4b -> 3403535d3
  refs/heads/branch-2.3 346f7086c -> 79339c158
  refs/heads/master 47c714167 -> 81be9d0d2


HIVE-16547 : LLAP: may not unlock buffers in some cases (Sergey Shelukhin, reviewed by Prasanth
Jayachandran)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f4d017b8
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f4d017b8
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f4d017b8

Branch: refs/heads/master
Commit: f4d017b801ea256bf076160a31dba88e61c80422
Parents: 47c7141
Author: sergey <sershe@apache.org>
Authored: Thu Apr 27 11:52:07 2017 -0700
Committer: sergey <sershe@apache.org>
Committed: Thu Apr 27 11:52:07 2017 -0700

----------------------------------------------------------------------
 .../ql/io/orc/encoded/EncodedReaderImpl.java    | 24 ++++++++++++++++----
 1 file changed, 19 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/f4d017b8/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
index 326b4b6..5b2e9b5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
@@ -416,8 +416,8 @@ class EncodedReaderImpl implements EncodedReader {
               if (sctx.stripeLevelStream == null) {
                 sctx.stripeLevelStream = POOLS.csdPool.take();
                 // We will be using this for each RG while also sending RGs to processing.
-                // To avoid buffers being unlocked, run refcount one ahead; we will not increase
-                // it when building the last RG, so each RG processing will decref once,
and the
+                // To avoid buffers being unlocked, run refcount one ahead; so each RG 
+                 // processing will decref once, and the
                 // last one will unlock the buffers.
                 sctx.stripeLevelStream.incRef();
                 // For stripe-level streams we don't need the extra refcount on the block.
@@ -430,9 +430,7 @@ class EncodedReaderImpl implements EncodedReader {
                   iter = lastCached;
                 }
               }
-              if (!isLastRg) {
-                sctx.stripeLevelStream.incRef();
-              }
+              sctx.stripeLevelStream.incRef();
               cb = sctx.stripeLevelStream;
             } else {
               // This stream can be separated by RG using index. Let's do that.
@@ -479,6 +477,22 @@ class EncodedReaderImpl implements EncodedReader {
     }
 
     // Release the unreleased buffers. See class comment about refcounts.
+    for (int colIx = 0; colIx < colCtxs.length; ++colIx) {
+      ColumnReadContext ctx = colCtxs[colIx];
+      if (ctx == null) continue; // This column is not included.
+      for (int streamIx = 0; streamIx < ctx.streamCount; ++streamIx) {
+        StreamContext sctx = ctx.streams[streamIx];
+        if (sctx == null || sctx.stripeLevelStream == null) continue;
+        if (0 != sctx.stripeLevelStream.decRef()) continue;
+        for (MemoryBuffer buf : sctx.stripeLevelStream.getCacheBuffers()) {
+          if (LOG.isTraceEnabled()) {
+            LOG.trace("Unlocking {} at the end of processing", buf);
+          }
+          cacheWrapper.releaseBuffer(buf);
+        }
+      }
+    }
+
     releaseInitialRefcounts(toRead.next);
     // Release buffers as we are done with all the streams... also see toRelease comment.
     if (toRelease != null) {


Mime
View raw message