hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From harisan...@apache.org
Subject hive git commit: HIVE-12084 : Hive queries with ORDER BY and large LIMIT fails with OutOfMemoryError Java heap space (Hari Subramaniyan, reviewed by Laljo John Pullokkaran)
Date Fri, 23 Oct 2015 17:54:15 GMT
Repository: hive
Updated Branches:
  refs/heads/branch-1 e35b908ea -> f4020cfce


HIVE-12084 : Hive queries with ORDER BY and large LIMIT fails with OutOfMemoryError Java heap
space (Hari Subramaniyan, reviewed by Laljo John Pullokkaran)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f4020cfc
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f4020cfc
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f4020cfc

Branch: refs/heads/branch-1
Commit: f4020cfce034cdcf91e362c106c15508b0ac16d8
Parents: e35b908
Author: Hari Subramaniyan <harisankar@apache.org>
Authored: Fri Oct 23 10:52:46 2015 -0700
Committer: Hari Subramaniyan <harisankar@apache.org>
Committed: Fri Oct 23 10:53:53 2015 -0700

----------------------------------------------------------------------
 .../apache/hadoop/hive/ql/exec/TopNHash.java    |  2 +-
 ql/src/test/queries/clientpositive/topn.q       | 13 ++++++
 ql/src/test/results/clientpositive/topn.q.out   | 42 ++++++++++++++++++++
 3 files changed, 56 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/f4020cfc/ql/src/java/org/apache/hadoop/hive/ql/exec/TopNHash.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/TopNHash.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/TopNHash.java
index 484006a..8859add 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/TopNHash.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/TopNHash.java
@@ -104,7 +104,7 @@ public class TopNHash {
     }
 
     // limit * 64 : compensation of arrays for key/value/hashcodes
-    this.threshold = (long) (memUsage * Runtime.getRuntime().maxMemory()) - topN * 64;
+    this.threshold = (long) (memUsage * Runtime.getRuntime().freeMemory()) - topN * 64L;
     if (threshold < 0) {
       return;
     }

http://git-wip-us.apache.org/repos/asf/hive/blob/f4020cfc/ql/src/test/queries/clientpositive/topn.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/topn.q b/ql/src/test/queries/clientpositive/topn.q
new file mode 100644
index 0000000..4821b5a
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/topn.q
@@ -0,0 +1,13 @@
+CREATE TABLE `sample_07` ( `code` string , `description` string , `total_emp` int , `salary`
int ) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' STORED AS TextFile;
+set hive.limit.pushdown.memory.usage=0.9999999;
+
+select * from sample_07 order by salary LIMIT 999999999;
+
+SELECT * FROM (
+SELECT *, rank() over(PARTITION BY code ORDER BY salary DESC) as rank
+FROM sample_07
+) ranked_claim
+WHERE ranked_claim.rank < 2
+ORDER BY code;
+
+select sum(total_emp) over(partition by salary+salary order by code) from sample_07 limit
9999999;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/f4020cfc/ql/src/test/results/clientpositive/topn.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/topn.q.out b/ql/src/test/results/clientpositive/topn.q.out
new file mode 100644
index 0000000..4846d8f
--- /dev/null
+++ b/ql/src/test/results/clientpositive/topn.q.out
@@ -0,0 +1,42 @@
+PREHOOK: query: CREATE TABLE `sample_07` ( `code` string , `description` string , `total_emp`
int , `salary` int ) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' STORED AS TextFile
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@sample_07
+POSTHOOK: query: CREATE TABLE `sample_07` ( `code` string , `description` string , `total_emp`
int , `salary` int ) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' STORED AS TextFile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@sample_07
+PREHOOK: query: select * from sample_07 order by salary LIMIT 999999999
+PREHOOK: type: QUERY
+PREHOOK: Input: default@sample_07
+#### A masked pattern was here ####
+POSTHOOK: query: select * from sample_07 order by salary LIMIT 999999999
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@sample_07
+#### A masked pattern was here ####
+PREHOOK: query: SELECT * FROM (
+SELECT *, rank() over(PARTITION BY code ORDER BY salary DESC) as rank
+FROM sample_07
+) ranked_claim
+WHERE ranked_claim.rank < 2
+ORDER BY code
+PREHOOK: type: QUERY
+PREHOOK: Input: default@sample_07
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM (
+SELECT *, rank() over(PARTITION BY code ORDER BY salary DESC) as rank
+FROM sample_07
+) ranked_claim
+WHERE ranked_claim.rank < 2
+ORDER BY code
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@sample_07
+#### A masked pattern was here ####
+PREHOOK: query: select sum(total_emp) over(partition by salary+salary order by code) from
sample_07 limit 9999999
+PREHOOK: type: QUERY
+PREHOOK: Input: default@sample_07
+#### A masked pattern was here ####
+POSTHOOK: query: select sum(total_emp) over(partition by salary+salary order by code) from
sample_07 limit 9999999
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@sample_07
+#### A masked pattern was here ####


Mime
View raw message