hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject hive git commit: HIVE-16012 : BytesBytes hash table - better capacity exhaustion handling (Sergey Shelukhin, reviewed by Wei Zheng)
Date Thu, 23 Feb 2017 19:30:54 GMT
Repository: hive
Updated Branches:
  refs/heads/master e941e63c7 -> 6ca79e3aa


HIVE-16012 : BytesBytes hash table - better capacity exhaustion handling (Sergey Shelukhin,
reviewed by Wei Zheng)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6ca79e3a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6ca79e3a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6ca79e3a

Branch: refs/heads/master
Commit: 6ca79e3aa2b5a8812e5c4aaee80c4115e2b9def8
Parents: e941e63
Author: Sergey Shelukhin <sershe@apache.org>
Authored: Thu Feb 23 11:25:18 2017 -0800
Committer: Sergey Shelukhin <sershe@apache.org>
Committed: Thu Feb 23 11:25:18 2017 -0800

----------------------------------------------------------------------
 .../hive/ql/exec/persistence/BytesBytesMultiHashMap.java  | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/6ca79e3a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/BytesBytesMultiHashMap.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/BytesBytesMultiHashMap.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/BytesBytesMultiHashMap.java
index 6b89e98..04e24bd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/BytesBytesMultiHashMap.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/BytesBytesMultiHashMap.java
@@ -26,6 +26,7 @@ import java.util.TreeMap;
 import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.ql.debug.Utils;
 import org.apache.hadoop.hive.serde2.ByteStream.RandomAccessOutput;
 import org.apache.hadoop.hive.serde2.SerDeException;
@@ -553,6 +554,12 @@ public final class BytesBytesMultiHashMap {
     if (capacity <= 0) {
       throw new AssertionError("Invalid capacity " + capacity);
     }
+    if (capacity > Integer.MAX_VALUE) {
+      throw new RuntimeException("Attempting to expand the hash table to " + capacity
+          + " that overflows maximum array size. For this query, you may want to disable
"
+          + ConfVars.HIVEDYNAMICPARTITIONHASHJOIN.varname + " or reduce "
+          + ConfVars.HIVECONVERTJOINNOCONDITIONALTASKTHRESHOLD.varname);
+    }
   }
 
   /**
@@ -715,8 +722,7 @@ public final class BytesBytesMultiHashMap {
   }
 
   private void expandAndRehash() {
-    long capacity = refs.length << 1;
-    expandAndRehashImpl(capacity);
+    expandAndRehashImpl(((long)refs.length) << 1);
   }
 
   private void expandAndRehashImpl(long capacity) {


Mime
View raw message