hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject [3/4] hive git commit: HIVE-15829 : LLAP text cache: disable memory tracking on the writer (Sergey Shelukhin, reviewed by Prasanth Jayachandran)
Date Tue, 28 Mar 2017 21:05:00 GMT
HIVE-15829 : LLAP text cache: disable memory tracking on the writer (Sergey Shelukhin, reviewed
by Prasanth Jayachandran)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0a10f959
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0a10f959
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0a10f959

Branch: refs/heads/master
Commit: 0a10f959f1745add8eab18d22bbae363186a04be
Parents: 5d83302
Author: Sergey Shelukhin <sershe@apache.org>
Authored: Tue Mar 28 13:57:52 2017 -0700
Committer: Sergey Shelukhin <sershe@apache.org>
Committed: Tue Mar 28 13:59:33 2017 -0700

----------------------------------------------------------------------
 .../llap/io/encoded/SerDeEncodedDataReader.java | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/0a10f959/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/SerDeEncodedDataReader.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/SerDeEncodedDataReader.java
b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/SerDeEncodedDataReader.java
index a4fde6d..907200a 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/SerDeEncodedDataReader.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/SerDeEncodedDataReader.java
@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.hive.llap.io.encoded;
 
+import org.apache.orc.impl.MemoryManager;
+
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.security.PrivilegedExceptionAction;
@@ -1532,13 +1534,29 @@ public class SerDeEncodedDataReader extends CallableWithNdc<Void>
     }
   }
 
+  private static final class NoopMemoryManager extends MemoryManager {
+    public NoopMemoryManager() {
+      super(null);
+    }
+
+    @Override
+    public void addedRow(int rows) {}
+    @Override
+    public void addWriter(Path path, long requestedAllocation, Callback callback) {}
+    @Override
+    public void notifyWriters() {}
+    @Override
+    public void removeWriter(Path path) throws IOException {}
+  }
+  private static final NoopMemoryManager MEMORY_MANAGER = new NoopMemoryManager();
+
   static WriterOptions createOrcWriterOptions(ObjectInspector sourceOi,
       Configuration conf, CacheWriter cacheWriter, int allocSize) throws IOException {
     return OrcFile.writerOptions(conf).stripeSize(Long.MAX_VALUE).blockSize(Long.MAX_VALUE)
         .rowIndexStride(Integer.MAX_VALUE) // For now, do not limit this - one RG per split
         .blockPadding(false).compress(CompressionKind.NONE).version(Version.CURRENT)
         .encodingStrategy(EncodingStrategy.SPEED).bloomFilterColumns(null).inspector(sourceOi)
-        .physicalWriter(cacheWriter).bufferSize(allocSize);
+        .physicalWriter(cacheWriter).memory(MEMORY_MANAGER).bufferSize(allocSize);
   }
 
   private ObjectInspector getOiFromSerDe() throws IOException {


Mime
View raw message