hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject hive git commit: HIVE-12956 : run CBO in tests with mapred.mode=strict (Ashutosh Chauhan via Jesus Camacho Rodriguez)
Date Sun, 31 Jan 2016 21:34:55 GMT
Repository: hive
Updated Branches:
  refs/heads/master 47f847188 -> 4d8b6c28b


HIVE-12956 : run CBO in tests with mapred.mode=strict (Ashutosh Chauhan via Jesus Camacho
Rodriguez)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4d8b6c28
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4d8b6c28
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4d8b6c28

Branch: refs/heads/master
Commit: 4d8b6c28b103f9456fd1fd026e9408424028c377
Parents: 47f8471
Author: Ashutosh Chauhan <hashutosh@apache.org>
Authored: Fri Jan 29 15:32:23 2016 -0800
Committer: Ashutosh Chauhan <hashutosh@apache.org>
Committed: Sun Jan 31 13:33:55 2016 -0800

----------------------------------------------------------------------
 .../hadoop/hive/ql/parse/CalcitePlanner.java    | 11 ++----
 .../test/results/clientpositive/groupby13.q.out | 20 +++++------
 .../test/results/clientpositive/input39.q.out   | 38 +++++++++++++-------
 .../test/results/clientpositive/input41.q.out   |  2 +-
 .../test/results/clientpositive/join_view.q.out | 36 +++++++++++--------
 .../clientpositive/spark/join_view.q.out        | 36 +++++++++++--------
 6 files changed, 83 insertions(+), 60 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/4d8b6c28/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index 3b3e840..e2d404b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -438,13 +438,8 @@ public class CalcitePlanner extends SemanticAnalyzer {
    */
   static String canHandleQbForCbo(QueryProperties queryProperties, HiveConf conf,
       boolean topLevelQB, boolean verbose, QB qb) {
-    boolean isInTest = conf.getBoolVar(ConfVars.HIVE_IN_TEST);
-    // TODO: HIVEMAPREDMODE is deprecated. Why does this test-only exception exist?
-    boolean isStrictTest = isInTest
-        && "strict".equals(HiveConf.getVar(conf, ConfVars.HIVEMAPREDMODE));
 
-    if (!isStrictTest
-        && !queryProperties.hasClusterBy() && !queryProperties.hasDistributeBy()
+    if (!queryProperties.hasClusterBy() && !queryProperties.hasDistributeBy()
         && !queryProperties.hasSortBy() && !queryProperties.hasPTF() &&
!queryProperties.usesScript()
         && !queryProperties.hasMultiDestQuery() && !queryProperties.hasLateralViews())
{
       // Ok to run CBO.
@@ -454,8 +449,6 @@ public class CalcitePlanner extends SemanticAnalyzer {
     // Not ok to run CBO, build error message.
     String msg = "";
     if (verbose) {
-      if (isStrictTest)
-        msg += "is in test running in strict mode (deprecated); ";
       if (queryProperties.hasClusterBy())
         msg += "has cluster by; ";
       if (queryProperties.hasDistributeBy())
@@ -1073,7 +1066,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
       perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER,
         "Calcite: Prejoin ordering transformation, factor out common filter elements and
separating deterministic vs non-deterministic UDF");
 
-      // 3. Run exhaustive PPD, add not null filters, transitive inference, 
+      // 3. Run exhaustive PPD, add not null filters, transitive inference,
       // constant propagation, constant folding
       perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER);
       basePlan = hepPlan(basePlan, true, mdProvider, executorProvider, HepMatchOrder.BOTTOM_UP,

http://git-wip-us.apache.org/repos/asf/hive/blob/4d8b6c28/ql/src/test/results/clientpositive/groupby13.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/groupby13.q.out b/ql/src/test/results/clientpositive/groupby13.q.out
index e698d1a..12bd925 100644
--- a/ql/src/test/results/clientpositive/groupby13.q.out
+++ b/ql/src/test/results/clientpositive/groupby13.q.out
@@ -44,25 +44,25 @@ STAGE PLANS:
             alias: grpby_test
             Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
             Select Operator
-              expressions: int_col_7 (type: int), int_col_5 (type: int)
-              outputColumnNames: int_col_7, int_col_5
+              expressions: int_col_7 (type: int), int_col_7 (type: int), least(COALESCE(int_col_5,-279),COALESCE(int_col_7,476))
(type: int)
+              outputColumnNames: _col0, _col1, _col2
               Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
               Group By Operator
-                aggregations: max(least(COALESCE(int_col_5,(- 279)),COALESCE(int_col_7,476)))
-                keys: int_col_7 (type: int), least(COALESCE(int_col_5,(- 279)),COALESCE(int_col_7,476))
(type: int)
+                aggregations: max(_col2)
+                keys: _col0 (type: int), _col1 (type: int), _col2 (type: int)
                 mode: hash
-                outputColumnNames: _col0, _col1, _col2
+                outputColumnNames: _col0, _col1, _col2, _col3
                 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
                 Reduce Output Operator
-                  key expressions: _col0 (type: int), _col1 (type: int)
-                  sort order: ++
-                  Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
+                  key expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int)
+                  sort order: +++
+                  Map-reduce partition columns: _col0 (type: int), _col1 (type: int), _col2
(type: int)
                   Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats:
NONE
-                  value expressions: _col2 (type: int)
+                  value expressions: _col3 (type: int)
       Reduce Operator Tree:
         Group By Operator
           aggregations: max(VALUE._col0)
-          keys: KEY._col0 (type: int), KEY._col0 (type: int), KEY._col1 (type: int)
+          keys: KEY._col0 (type: int), KEY._col1 (type: int), KEY._col2 (type: int)
           mode: mergepartial
           outputColumnNames: _col0, _col1, _col2, _col3
           Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE

http://git-wip-us.apache.org/repos/asf/hive/blob/4d8b6c28/ql/src/test/results/clientpositive/input39.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/input39.q.out b/ql/src/test/results/clientpositive/input39.q.out
index 9e8e4f9..dd69f33 100644
--- a/ql/src/test/results/clientpositive/input39.q.out
+++ b/ql/src/test/results/clientpositive/input39.q.out
@@ -75,31 +75,45 @@ STAGE PLANS:
             alias: t1
             Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats:
NONE
             Filter Operator
-              predicate: ((((hash(rand(460476415)) & 2147483647) % 32) = 0) and key is
not null) (type: boolean)
+              predicate: (((hash(rand(460476415)) & 2147483647) % 32) = 0) (type: boolean)
               Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats:
NONE
-              Reduce Output Operator
-                key expressions: key (type: string)
-                sort order: +
-                Map-reduce partition columns: key (type: string)
+              Filter Operator
+                predicate: key is not null (type: boolean)
                 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats:
NONE
+                Select Operator
+                  expressions: key (type: string)
+                  outputColumnNames: _col0
+                  Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column
stats: NONE
+                  Reduce Output Operator
+                    key expressions: _col0 (type: string)
+                    sort order: +
+                    Map-reduce partition columns: _col0 (type: string)
+                    Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column
stats: NONE
           TableScan
             alias: t2
             Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats:
NONE
             Filter Operator
-              predicate: ((((hash(rand(460476415)) & 2147483647) % 32) = 0) and key is
not null) (type: boolean)
+              predicate: (((hash(rand(460476415)) & 2147483647) % 32) = 0) (type: boolean)
               Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats:
NONE
-              Reduce Output Operator
-                key expressions: key (type: string)
-                sort order: +
-                Map-reduce partition columns: key (type: string)
+              Filter Operator
+                predicate: key is not null (type: boolean)
                 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats:
NONE
+                Select Operator
+                  expressions: key (type: string)
+                  outputColumnNames: _col0
+                  Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column
stats: NONE
+                  Reduce Output Operator
+                    key expressions: _col0 (type: string)
+                    sort order: +
+                    Map-reduce partition columns: _col0 (type: string)
+                    Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column
stats: NONE
       Reduce Operator Tree:
         Join Operator
           condition map:
                Inner Join 0 to 1
           keys:
-            0 key (type: string)
-            1 key (type: string)
+            0 _col0 (type: string)
+            1 _col0 (type: string)
           Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
           Group By Operator
             aggregations: count(1)

http://git-wip-us.apache.org/repos/asf/hive/blob/4d8b6c28/ql/src/test/results/clientpositive/input41.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/input41.q.out b/ql/src/test/results/clientpositive/input41.q.out
index 7cc1007..871f198 100644
--- a/ql/src/test/results/clientpositive/input41.q.out
+++ b/ql/src/test/results/clientpositive/input41.q.out
@@ -26,7 +26,7 @@ POSTHOOK: type: QUERY
 POSTHOOK: Input: default@src
 POSTHOOK: Input: default@srcpart
 POSTHOOK: Output: default@dest_sp
-POSTHOOK: Lineage: dest_sp.cnt EXPRESSION [(src)src.null, (srcpart)srcpart.null, ]
+POSTHOOK: Lineage: dest_sp.cnt EXPRESSION []
 PREHOOK: query: select * from dest_sp x order by x.cnt limit 2
 PREHOOK: type: QUERY
 PREHOOK: Input: default@dest_sp

http://git-wip-us.apache.org/repos/asf/hive/blob/4d8b6c28/ql/src/test/results/clientpositive/join_view.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/join_view.q.out b/ql/src/test/results/clientpositive/join_view.q.out
index 085ae60..5ec1806 100644
--- a/ql/src/test/results/clientpositive/join_view.q.out
+++ b/ql/src/test/results/clientpositive/join_view.q.out
@@ -56,35 +56,43 @@ STAGE PLANS:
             Filter Operator
               predicate: (ds = '2011-09-01') (type: boolean)
               Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
-              Reduce Output Operator
-                key expressions: '2011-09-01' (type: string)
-                sort order: +
-                Map-reduce partition columns: '2011-09-01' (type: string)
+              Select Operator
+                expressions: bar (type: string)
+                outputColumnNames: _col0
                 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
-                value expressions: bar (type: string)
+                Reduce Output Operator
+                  key expressions: '2011-09-01' (type: string)
+                  sort order: +
+                  Map-reduce partition columns: '2011-09-01' (type: string)
+                  Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats:
NONE
+                  value expressions: _col0 (type: string)
           TableScan
             alias: invites2
             Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
             Filter Operator
               predicate: (ds = '2011-09-01') (type: boolean)
               Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
-              Reduce Output Operator
-                key expressions: '2011-09-01' (type: string)
-                sort order: +
-                Map-reduce partition columns: '2011-09-01' (type: string)
+              Select Operator
+                expressions: foo (type: int)
+                outputColumnNames: _col0
                 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
-                value expressions: foo (type: int)
+                Reduce Output Operator
+                  key expressions: '2011-09-01' (type: string)
+                  sort order: +
+                  Map-reduce partition columns: '2011-09-01' (type: string)
+                  Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats:
NONE
+                  value expressions: _col0 (type: int)
       Reduce Operator Tree:
         Join Operator
           condition map:
                Inner Join 0 to 1
           keys:
-            0 ds (type: string)
-            1 ds (type: string)
-          outputColumnNames: _col1, _col6
+            0 _col1 (type: string)
+            1 _col1 (type: string)
+          outputColumnNames: _col0, _col2
           Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
           Select Operator
-            expressions: _col1 (type: string), _col6 (type: int), '2011-09-01' (type: string)
+            expressions: _col0 (type: string), _col2 (type: int), '2011-09-01' (type: string)
             outputColumnNames: _col0, _col1, _col2
             Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
             File Output Operator

http://git-wip-us.apache.org/repos/asf/hive/blob/4d8b6c28/ql/src/test/results/clientpositive/spark/join_view.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/join_view.q.out b/ql/src/test/results/clientpositive/spark/join_view.q.out
index 8a51771..5fe0929 100644
--- a/ql/src/test/results/clientpositive/spark/join_view.q.out
+++ b/ql/src/test/results/clientpositive/spark/join_view.q.out
@@ -61,12 +61,16 @@ STAGE PLANS:
                   Filter Operator
                     predicate: (ds = '2011-09-01') (type: boolean)
                     Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats:
NONE
-                    Reduce Output Operator
-                      key expressions: '2011-09-01' (type: string)
-                      sort order: +
-                      Map-reduce partition columns: '2011-09-01' (type: string)
+                    Select Operator
+                      expressions: bar (type: string)
+                      outputColumnNames: _col0
                       Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats:
NONE
-                      value expressions: bar (type: string)
+                      Reduce Output Operator
+                        key expressions: '2011-09-01' (type: string)
+                        sort order: +
+                        Map-reduce partition columns: '2011-09-01' (type: string)
+                        Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column
stats: NONE
+                        value expressions: _col0 (type: string)
         Map 3 
             Map Operator Tree:
                 TableScan
@@ -75,24 +79,28 @@ STAGE PLANS:
                   Filter Operator
                     predicate: (ds = '2011-09-01') (type: boolean)
                     Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats:
NONE
-                    Reduce Output Operator
-                      key expressions: '2011-09-01' (type: string)
-                      sort order: +
-                      Map-reduce partition columns: '2011-09-01' (type: string)
+                    Select Operator
+                      expressions: foo (type: int)
+                      outputColumnNames: _col0
                       Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats:
NONE
-                      value expressions: foo (type: int)
+                      Reduce Output Operator
+                        key expressions: '2011-09-01' (type: string)
+                        sort order: +
+                        Map-reduce partition columns: '2011-09-01' (type: string)
+                        Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column
stats: NONE
+                        value expressions: _col0 (type: int)
         Reducer 2 
             Reduce Operator Tree:
               Join Operator
                 condition map:
                      Inner Join 0 to 1
                 keys:
-                  0 ds (type: string)
-                  1 ds (type: string)
-                outputColumnNames: _col1, _col6
+                  0 _col1 (type: string)
+                  1 _col1 (type: string)
+                outputColumnNames: _col0, _col2
                 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
                 Select Operator
-                  expressions: _col1 (type: string), _col6 (type: int), '2011-09-01' (type:
string)
+                  expressions: _col0 (type: string), _col2 (type: int), '2011-09-01' (type:
string)
                   outputColumnNames: _col0, _col1, _col2
                   Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats:
NONE
                   File Output Operator


Mime
View raw message