hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject [44/50] [abbrv] hive git commit: HIVE-16068: BloomFilter expectedEntries not always using NDV when it's available during runtime filtering (Jason Dere, reviewed by Gunther Hagleitner)
Date Tue, 28 Mar 2017 22:32:12 GMT
HIVE-16068: BloomFilter expectedEntries not always using NDV when it's available during runtime
filtering (Jason Dere, reviewed by Gunther Hagleitner)

Change-Id: I9a5f3960e1b7fc1416f510aefbeab6f28bbafffa


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/43906f9c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/43906f9c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/43906f9c

Branch: refs/heads/branch-2.2
Commit: 43906f9c3dbe2e63ffc744b69925b753a119e8be
Parents: 72ca985
Author: Jason Dere <jdere@hortonworks.com>
Authored: Fri Mar 3 11:53:02 2017 -0800
Committer: Owen O'Malley <omalley@apache.org>
Committed: Tue Mar 28 15:27:57 2017 -0700

----------------------------------------------------------------------
 .../hive/ql/udf/generic/GenericUDAFBloomFilter.java | 16 +++++++++++-----
 .../results/clientpositive/llap/mergejoin.q.out     |  4 ++--
 .../clientpositive/tez/explainanalyze_3.q.out       |  4 ++--
 3 files changed, 15 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/43906f9c/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFBloomFilter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFBloomFilter.java
b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFBloomFilter.java
index b32e04a..788aace 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFBloomFilter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFBloomFilter.java
@@ -20,10 +20,13 @@ package org.apache.hadoop.hive.ql.udf.generic;
 
 import org.apache.hadoop.hive.common.type.HiveDecimal;
 import org.apache.hadoop.hive.ql.exec.Operator;
+import org.apache.hadoop.hive.ql.exec.SelectOperator;
 import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.parse.SemanticException;
 import org.apache.hadoop.hive.ql.plan.ColStatistics;
+import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils;
 import org.apache.hadoop.hive.ql.plan.Statistics;
 import org.apache.hadoop.hive.ql.plan.Statistics.State;
 import org.apache.hadoop.hive.serde2.io.DateWritable;
@@ -68,7 +71,7 @@ public class GenericUDAFBloomFilter implements GenericUDAFResolver2 {
    */
   public static class GenericUDAFBloomFilterEvaluator extends GenericUDAFEvaluator {
     // Source operator to get the number of entries
-    private Operator<?> sourceOperator;
+    private SelectOperator sourceOperator;
     private long maxEntries = 0;
 
     // ObjectInspector for input data.
@@ -258,10 +261,13 @@ public class GenericUDAFBloomFilter implements GenericUDAFResolver2
{
         switch (stats.getColumnStatsState()) {
           case COMPLETE:
           case PARTIAL:
-            // There should only be column stats for one column, use if that is the case.
+            // There should only be column in sourceOperator
             List<ColStatistics> colStats = stats.getColumnStats();
-            if (colStats.size() == 1) {
-              long ndv = colStats.get(0).getCountDistint();
+            ExprNodeColumnDesc colExpr = ExprNodeDescUtils.getColumnExpr(
+                sourceOperator.getConf().getColList().get(0));
+            if (colExpr != null
+                && stats.getColumnStatisticsFromColName(colExpr.getColumn()) != null)
{
+              long ndv = stats.getColumnStatisticsFromColName(colExpr.getColumn()).getCountDistint();
               if (ndv > 0) {
                 expectedEntries = ndv;
               }
@@ -279,7 +285,7 @@ public class GenericUDAFBloomFilter implements GenericUDAFResolver2 {
       return sourceOperator;
     }
 
-    public void setSourceOperator(Operator<?> sourceOperator) {
+    public void setSourceOperator(SelectOperator sourceOperator) {
       this.sourceOperator = sourceOperator;
     }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/43906f9c/ql/src/test/results/clientpositive/llap/mergejoin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/mergejoin.q.out b/ql/src/test/results/clientpositive/llap/mergejoin.q.out
index 85f14a0..4e6e2b4 100644
--- a/ql/src/test/results/clientpositive/llap/mergejoin.q.out
+++ b/ql/src/test/results/clientpositive/llap/mergejoin.q.out
@@ -63,7 +63,7 @@ STAGE PLANS:
                         outputColumnNames: _col0
                         Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column
stats: NONE
                         Group By Operator
-                          aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=25)
+                          aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=14)
                           mode: hash
                           outputColumnNames: _col0, _col1, _col2
                           Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column
stats: NONE
@@ -95,7 +95,7 @@ STAGE PLANS:
             Execution mode: vectorized, llap
             Reduce Operator Tree:
               Group By Operator
-                aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2,
expectedEntries=25)
+                aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2,
expectedEntries=14)
                 mode: final
                 outputColumnNames: _col0, _col1, _col2
                 Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats:
NONE

http://git-wip-us.apache.org/repos/asf/hive/blob/43906f9c/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out b/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out
index dfe163e..14fac5f 100644
--- a/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out
+++ b/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out
@@ -869,10 +869,10 @@ STAGE PLANS:
             Map Operator Tree:
                 TableScan
                   alias: b
-                  filterExpr: (key is not null and key BETWEEN DynamicValue(RS_6_a_key_min)
AND DynamicValue(RS_6_a_key_max) and in_bloom_filter(key, DynamicValue(RS_6_a_key_bloom_filter)))
(type: boolean)
+                  filterExpr: (key is not null and (key BETWEEN DynamicValue(RS_6_a_key_min)
AND DynamicValue(RS_6_a_key_max) and in_bloom_filter(key, DynamicValue(RS_6_a_key_bloom_filter))))
(type: boolean)
                   Statistics: Num rows: 500/500 Data size: 5312 Basic stats: COMPLETE Column
stats: NONE
                   Filter Operator
-                    predicate: (key is not null and key BETWEEN DynamicValue(RS_6_a_key_min)
AND DynamicValue(RS_6_a_key_max) and in_bloom_filter(key, DynamicValue(RS_6_a_key_bloom_filter)))
(type: boolean)
+                    predicate: (key is not null and (key BETWEEN DynamicValue(RS_6_a_key_min)
AND DynamicValue(RS_6_a_key_max) and in_bloom_filter(key, DynamicValue(RS_6_a_key_bloom_filter))))
(type: boolean)
                     Statistics: Num rows: 500/244 Data size: 5312 Basic stats: COMPLETE Column
stats: NONE
                     Select Operator
                       expressions: key (type: int), value (type: string)


Mime
View raw message