hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gop...@apache.org
Subject [73/82] [abbrv] hive git commit: HIVE-10711 Tez HashTableLoader attempts to allocate more memory than available (Mostafa Mokhtar and Jason Dere via Alexander Pivovarov)
Date Fri, 29 May 2015 00:51:20 GMT
HIVE-10711 Tez HashTableLoader attempts to allocate more memory than available (Mostafa Mokhtar
and Jason Dere via Alexander Pivovarov)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1293f3d3
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1293f3d3
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1293f3d3

Branch: refs/heads/llap
Commit: 1293f3d389b73eea672805c49270124f44cbd687
Parents: ac25506
Author: Alexander Pivovarov <apivovarov@gmail.com>
Authored: Wed May 27 23:30:39 2015 -0700
Committer: Alexander Pivovarov <apivovarov@gmail.com>
Committed: Wed May 27 23:30:39 2015 -0700

----------------------------------------------------------------------
 .../apache/hadoop/hive/ql/exec/tez/HashTableLoader.java   | 10 ++++++++++
 1 file changed, 10 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/1293f3d3/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
index 536b92c..7e67d40 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
@@ -18,6 +18,7 @@
 package org.apache.hadoop.hive.ql.exec.tez;
 
 import java.io.IOException;
+import java.lang.management.ManagementFactory;
 import java.util.Collections;
 import java.util.Map;
 
@@ -82,6 +83,15 @@ public class HashTableLoader implements org.apache.hadoop.hive.ql.exec.HashTable
     // TODO remove this after memory manager is in
     long noConditionalTaskThreshold = HiveConf.getLongVar(
         hconf, HiveConf.ConfVars.HIVECONVERTJOINNOCONDITIONALTASKTHRESHOLD);
+    long processMaxMemory = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax();
+    if (noConditionalTaskThreshold > processMaxMemory) {
+      float hashtableMemoryUsage = HiveConf.getFloatVar(
+          hconf, HiveConf.ConfVars.HIVEHASHTABLEFOLLOWBYGBYMAXMEMORYUSAGE);
+      LOG.warn("noConditionalTaskThreshold value of " + noConditionalTaskThreshold +
+          " is greater than the max memory size of " + processMaxMemory);
+      // Don't want to attempt to grab more memory than we have available .. percentage is
a bit arbitrary
+      noConditionalTaskThreshold = (long) (processMaxMemory * hashtableMemoryUsage);
+    }
 
     // Only applicable to n-way Hybrid Grace Hash Join
     HybridHashTableConf nwayConf = null;


Mime
View raw message