hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From prasan...@apache.org
Subject hive git commit: HIVE-12735: Constant folding for WHEN/CASE expression does not set return type correctly
Date Thu, 24 Dec 2015 02:46:20 GMT
Repository: hive
Updated Branches:
  refs/heads/master f677bffa9 -> 07dfecee7


HIVE-12735: Constant folding for WHEN/CASE expression does not set return type correctly


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/07dfecee
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/07dfecee
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/07dfecee

Branch: refs/heads/master
Commit: 07dfecee7a5b8bbdae1a2ea371b3ea9fc8046930
Parents: f677bff
Author: Prasanth Jayachandran <j.prasanth.j@gmail.com>
Authored: Wed Dec 23 20:46:01 2015 -0600
Committer: Prasanth Jayachandran <j.prasanth.j@gmail.com>
Committed: Wed Dec 23 20:46:01 2015 -0600

----------------------------------------------------------------------
 .../optimizer/ConstantPropagateProcFactory.java |   12 +-
 .../clientpositive/constprog_when_case.q        |   12 +
 .../clientpositive/constprog_when_case.q.out    | 1074 ++++++++++++++++++
 .../clientpositive/fold_eq_with_case_when.q.out |    4 +-
 4 files changed, 1098 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/07dfecee/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java
index b18b5af..d182d80 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java
@@ -568,7 +568,11 @@ public final class ConstantPropagateProcFactory {
          children.set(i, ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPEqual(),
              Lists.newArrayList(children.get(i),newExprs.get(foundUDFInFirst ? 1 : 0))));
        }
-       return caseOrWhenexpr;
+       // after constant folding of child expression the return type of UDFWhen might have
changed,
+       // so recreate the expression
+       ExprNodeGenericFuncDesc newCaseOrWhenExpr = ExprNodeGenericFuncDesc.newInstance(childUDF,
+           caseOrWhenexpr.getFuncText(), children);
+       return newCaseOrWhenExpr;
      } else if (childUDF instanceof GenericUDFCase) {
        for (i = 2; i < children.size(); i+=2) {
          children.set(i, ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPEqual(),
@@ -579,7 +583,11 @@ public final class ConstantPropagateProcFactory {
           children.set(i, ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPEqual(),
               Lists.newArrayList(children.get(i),newExprs.get(foundUDFInFirst ? 1 : 0))));
         }
-        return caseOrWhenexpr;
+       // after constant folding of child expression the return type of UDFCase might have
changed,
+       // so recreate the expression
+       ExprNodeGenericFuncDesc newCaseOrWhenExpr = ExprNodeGenericFuncDesc.newInstance(childUDF,
+           caseOrWhenexpr.getFuncText(), children);
+       return newCaseOrWhenExpr;
      } else {
        // cant happen
        return null;

http://git-wip-us.apache.org/repos/asf/hive/blob/07dfecee/ql/src/test/queries/clientpositive/constprog_when_case.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/constprog_when_case.q b/ql/src/test/queries/clientpositive/constprog_when_case.q
new file mode 100644
index 0000000..b2a10f3
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/constprog_when_case.q
@@ -0,0 +1,12 @@
+set hive.fetch.task.conversion=none;
+set hive.explain.user=false;
+set hive.vectorized.execution.enabled=true;
+
+create table src_orc (bool0 boolean, key0 string, key1 string, key2 string) stored as orc;
+insert overwrite table src_orc select true, key, value, key from src;
+insert into table src_orc select true, key, value, key from src;
+
+explain SELECT IF ( ( (CASE WHEN bool0 THEN 1 WHEN NOT bool0 THEN 0 END) = (CASE WHEN TRUE
THEN 1 WHEN NOT TRUE THEN 0 END) ), key0, IF ( ( (CASE WHEN bool0 THEN 1 WHEN NOT bool0 THEN
0 END) = (CASE WHEN FALSE THEN 1 WHEN NOT FALSE THEN 0 END) ), key1, key2 ) ) FROM src_orc;
+
+SELECT IF ( ( (CASE WHEN bool0 THEN 1 WHEN NOT bool0 THEN 0 END) = (CASE WHEN TRUE THEN 1
WHEN NOT TRUE THEN 0 END) ), key0, IF ( ( (CASE WHEN bool0 THEN 1 WHEN NOT bool0 THEN 0 END)
= (CASE WHEN FALSE THEN 1 WHEN NOT FALSE THEN 0 END) ), key1, key2 ) ) FROM src_orc;
+

http://git-wip-us.apache.org/repos/asf/hive/blob/07dfecee/ql/src/test/results/clientpositive/constprog_when_case.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/constprog_when_case.q.out b/ql/src/test/results/clientpositive/constprog_when_case.q.out
new file mode 100644
index 0000000..7f8a504
--- /dev/null
+++ b/ql/src/test/results/clientpositive/constprog_when_case.q.out
@@ -0,0 +1,1074 @@
+PREHOOK: query: create table src_orc (bool0 boolean, key0 string, key1 string, key2 string)
stored as orc
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@src_orc
+POSTHOOK: query: create table src_orc (bool0 boolean, key0 string, key1 string, key2 string)
stored as orc
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@src_orc
+PREHOOK: query: insert overwrite table src_orc select true, key, value, key from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@src_orc
+POSTHOOK: query: insert overwrite table src_orc select true, key, value, key from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@src_orc
+POSTHOOK: Lineage: src_orc.bool0 SIMPLE []
+POSTHOOK: Lineage: src_orc.key0 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default),
]
+POSTHOOK: Lineage: src_orc.key1 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default),
]
+POSTHOOK: Lineage: src_orc.key2 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default),
]
+PREHOOK: query: insert into table src_orc select true, key, value, key from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@src_orc
+POSTHOOK: query: insert into table src_orc select true, key, value, key from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@src_orc
+POSTHOOK: Lineage: src_orc.bool0 SIMPLE []
+POSTHOOK: Lineage: src_orc.key0 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default),
]
+POSTHOOK: Lineage: src_orc.key1 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default),
]
+POSTHOOK: Lineage: src_orc.key2 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default),
]
+PREHOOK: query: explain SELECT IF ( ( (CASE WHEN bool0 THEN 1 WHEN NOT bool0 THEN 0 END)
= (CASE WHEN TRUE THEN 1 WHEN NOT TRUE THEN 0 END) ), key0, IF ( ( (CASE WHEN bool0 THEN 1
WHEN NOT bool0 THEN 0 END) = (CASE WHEN FALSE THEN 1 WHEN NOT FALSE THEN 0 END) ), key1, key2
) ) FROM src_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain SELECT IF ( ( (CASE WHEN bool0 THEN 1 WHEN NOT bool0 THEN 0 END)
= (CASE WHEN TRUE THEN 1 WHEN NOT TRUE THEN 0 END) ), key0, IF ( ( (CASE WHEN bool0 THEN 1
WHEN NOT bool0 THEN 0 END) = (CASE WHEN FALSE THEN 1 WHEN NOT FALSE THEN 0 END) ), key1, key2
) ) FROM src_orc
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Map Operator Tree:
+          TableScan
+            alias: src_orc
+            Statistics: Num rows: 1000 Data size: 266000 Basic stats: COMPLETE Column stats:
NONE
+            Select Operator
+              expressions: if(CASE WHEN (bool0) THEN (true) WHEN ((not bool0)) THEN (false)
END, key0, if(CASE WHEN (bool0) THEN (false) WHEN ((not bool0)) THEN (true) END, key1, key2))
(type: string)
+              outputColumnNames: _col0
+              Statistics: Num rows: 1000 Data size: 266000 Basic stats: COMPLETE Column stats:
NONE
+              File Output Operator
+                compressed: false
+                Statistics: Num rows: 1000 Data size: 266000 Basic stats: COMPLETE Column
stats: NONE
+                table:
+                    input format: org.apache.hadoop.mapred.TextInputFormat
+                    output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                    serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+      Execution mode: vectorized
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+
+PREHOOK: query: SELECT IF ( ( (CASE WHEN bool0 THEN 1 WHEN NOT bool0 THEN 0 END) = (CASE
WHEN TRUE THEN 1 WHEN NOT TRUE THEN 0 END) ), key0, IF ( ( (CASE WHEN bool0 THEN 1 WHEN NOT
bool0 THEN 0 END) = (CASE WHEN FALSE THEN 1 WHEN NOT FALSE THEN 0 END) ), key1, key2 ) ) FROM
src_orc
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_orc
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT IF ( ( (CASE WHEN bool0 THEN 1 WHEN NOT bool0 THEN 0 END) = (CASE
WHEN TRUE THEN 1 WHEN NOT TRUE THEN 0 END) ), key0, IF ( ( (CASE WHEN bool0 THEN 1 WHEN NOT
bool0 THEN 0 END) = (CASE WHEN FALSE THEN 1 WHEN NOT FALSE THEN 0 END) ), key1, key2 ) ) FROM
src_orc
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_orc
+#### A masked pattern was here ####
+238
+86
+311
+27
+165
+409
+255
+278
+98
+484
+265
+193
+401
+150
+273
+224
+369
+66
+128
+213
+146
+406
+429
+374
+152
+469
+145
+495
+37
+327
+281
+277
+209
+15
+82
+403
+166
+417
+430
+252
+292
+219
+287
+153
+193
+338
+446
+459
+394
+237
+482
+174
+413
+494
+207
+199
+466
+208
+174
+399
+396
+247
+417
+489
+162
+377
+397
+309
+365
+266
+439
+342
+367
+325
+167
+195
+475
+17
+113
+155
+203
+339
+0
+455
+128
+311
+316
+57
+302
+205
+149
+438
+345
+129
+170
+20
+489
+157
+378
+221
+92
+111
+47
+72
+4
+280
+35
+427
+277
+208
+356
+399
+169
+382
+498
+125
+386
+437
+469
+192
+286
+187
+176
+54
+459
+51
+138
+103
+239
+213
+216
+430
+278
+176
+289
+221
+65
+318
+332
+311
+275
+137
+241
+83
+333
+180
+284
+12
+230
+181
+67
+260
+404
+384
+489
+353
+373
+272
+138
+217
+84
+348
+466
+58
+8
+411
+230
+208
+348
+24
+463
+431
+179
+172
+42
+129
+158
+119
+496
+0
+322
+197
+468
+393
+454
+100
+298
+199
+191
+418
+96
+26
+165
+327
+230
+205
+120
+131
+51
+404
+43
+436
+156
+469
+468
+308
+95
+196
+288
+481
+457
+98
+282
+197
+187
+318
+318
+409
+470
+137
+369
+316
+169
+413
+85
+77
+0
+490
+87
+364
+179
+118
+134
+395
+282
+138
+238
+419
+15
+118
+72
+90
+307
+19
+435
+10
+277
+273
+306
+224
+309
+389
+327
+242
+369
+392
+272
+331
+401
+242
+452
+177
+226
+5
+497
+402
+396
+317
+395
+58
+35
+336
+95
+11
+168
+34
+229
+233
+143
+472
+322
+498
+160
+195
+42
+321
+430
+119
+489
+458
+78
+76
+41
+223
+492
+149
+449
+218
+228
+138
+453
+30
+209
+64
+468
+76
+74
+342
+69
+230
+33
+368
+103
+296
+113
+216
+367
+344
+167
+274
+219
+239
+485
+116
+223
+256
+263
+70
+487
+480
+401
+288
+191
+5
+244
+438
+128
+467
+432
+202
+316
+229
+469
+463
+280
+2
+35
+283
+331
+235
+80
+44
+193
+321
+335
+104
+466
+366
+175
+403
+483
+53
+105
+257
+406
+409
+190
+406
+401
+114
+258
+90
+203
+262
+348
+424
+12
+396
+201
+217
+164
+431
+454
+478
+298
+125
+431
+164
+424
+187
+382
+5
+70
+397
+480
+291
+24
+351
+255
+104
+70
+163
+438
+119
+414
+200
+491
+237
+439
+360
+248
+479
+305
+417
+199
+444
+120
+429
+169
+443
+323
+325
+277
+230
+478
+178
+468
+310
+317
+333
+493
+460
+207
+249
+265
+480
+83
+136
+353
+172
+214
+462
+233
+406
+133
+175
+189
+454
+375
+401
+421
+407
+384
+256
+26
+134
+67
+384
+379
+18
+462
+492
+100
+298
+9
+341
+498
+146
+458
+362
+186
+285
+348
+167
+18
+273
+183
+281
+344
+97
+469
+315
+84
+28
+37
+448
+152
+348
+307
+194
+414
+477
+222
+126
+90
+169
+403
+400
+200
+97
+238
+86
+311
+27
+165
+409
+255
+278
+98
+484
+265
+193
+401
+150
+273
+224
+369
+66
+128
+213
+146
+406
+429
+374
+152
+469
+145
+495
+37
+327
+281
+277
+209
+15
+82
+403
+166
+417
+430
+252
+292
+219
+287
+153
+193
+338
+446
+459
+394
+237
+482
+174
+413
+494
+207
+199
+466
+208
+174
+399
+396
+247
+417
+489
+162
+377
+397
+309
+365
+266
+439
+342
+367
+325
+167
+195
+475
+17
+113
+155
+203
+339
+0
+455
+128
+311
+316
+57
+302
+205
+149
+438
+345
+129
+170
+20
+489
+157
+378
+221
+92
+111
+47
+72
+4
+280
+35
+427
+277
+208
+356
+399
+169
+382
+498
+125
+386
+437
+469
+192
+286
+187
+176
+54
+459
+51
+138
+103
+239
+213
+216
+430
+278
+176
+289
+221
+65
+318
+332
+311
+275
+137
+241
+83
+333
+180
+284
+12
+230
+181
+67
+260
+404
+384
+489
+353
+373
+272
+138
+217
+84
+348
+466
+58
+8
+411
+230
+208
+348
+24
+463
+431
+179
+172
+42
+129
+158
+119
+496
+0
+322
+197
+468
+393
+454
+100
+298
+199
+191
+418
+96
+26
+165
+327
+230
+205
+120
+131
+51
+404
+43
+436
+156
+469
+468
+308
+95
+196
+288
+481
+457
+98
+282
+197
+187
+318
+318
+409
+470
+137
+369
+316
+169
+413
+85
+77
+0
+490
+87
+364
+179
+118
+134
+395
+282
+138
+238
+419
+15
+118
+72
+90
+307
+19
+435
+10
+277
+273
+306
+224
+309
+389
+327
+242
+369
+392
+272
+331
+401
+242
+452
+177
+226
+5
+497
+402
+396
+317
+395
+58
+35
+336
+95
+11
+168
+34
+229
+233
+143
+472
+322
+498
+160
+195
+42
+321
+430
+119
+489
+458
+78
+76
+41
+223
+492
+149
+449
+218
+228
+138
+453
+30
+209
+64
+468
+76
+74
+342
+69
+230
+33
+368
+103
+296
+113
+216
+367
+344
+167
+274
+219
+239
+485
+116
+223
+256
+263
+70
+487
+480
+401
+288
+191
+5
+244
+438
+128
+467
+432
+202
+316
+229
+469
+463
+280
+2
+35
+283
+331
+235
+80
+44
+193
+321
+335
+104
+466
+366
+175
+403
+483
+53
+105
+257
+406
+409
+190
+406
+401
+114
+258
+90
+203
+262
+348
+424
+12
+396
+201
+217
+164
+431
+454
+478
+298
+125
+431
+164
+424
+187
+382
+5
+70
+397
+480
+291
+24
+351
+255
+104
+70
+163
+438
+119
+414
+200
+491
+237
+439
+360
+248
+479
+305
+417
+199
+444
+120
+429
+169
+443
+323
+325
+277
+230
+478
+178
+468
+310
+317
+333
+493
+460
+207
+249
+265
+480
+83
+136
+353
+172
+214
+462
+233
+406
+133
+175
+189
+454
+375
+401
+421
+407
+384
+256
+26
+134
+67
+384
+379
+18
+462
+492
+100
+298
+9
+341
+498
+146
+458
+362
+186
+285
+348
+167
+18
+273
+183
+281
+344
+97
+469
+315
+84
+28
+37
+448
+152
+348
+307
+194
+414
+477
+222
+126
+90
+169
+403
+400
+200
+97

http://git-wip-us.apache.org/repos/asf/hive/blob/07dfecee/ql/src/test/results/clientpositive/fold_eq_with_case_when.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/fold_eq_with_case_when.q.out b/ql/src/test/results/clientpositive/fold_eq_with_case_when.q.out
index 231d98f..69dd050 100644
--- a/ql/src/test/results/clientpositive/fold_eq_with_case_when.q.out
+++ b/ql/src/test/results/clientpositive/fold_eq_with_case_when.q.out
@@ -137,7 +137,7 @@ STAGE PLANS:
             alias: src
             Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats:
NONE
             Filter Operator
-              predicate: CASE (key) WHEN ('238') THEN (true) WHEN ('94') THEN (true) ELSE
(false) END (type: int)
+              predicate: CASE (key) WHEN ('238') THEN (true) WHEN ('94') THEN (true) ELSE
(false) END (type: boolean)
               Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats:
NONE
               Select Operator
                 expressions: key (type: string)
@@ -209,7 +209,7 @@ STAGE PLANS:
             alias: src
             Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats:
NONE
             Filter Operator
-              predicate: CASE (key) WHEN ('238') THEN (CASE WHEN ((key <> '238')) THEN
(true) WHEN ((key = '23')) THEN (true) END) END (type: int)
+              predicate: CASE (key) WHEN ('238') THEN (CASE WHEN ((key <> '238')) THEN
(true) WHEN ((key = '23')) THEN (true) END) END (type: boolean)
               Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats:
NONE
               Select Operator
                 expressions: key (type: string)


Mime
View raw message