hive-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Pengcheng Xiong (JIRA)" <j...@apache.org>
Subject [jira] [Comment Edited] (HIVE-16485) Enable outputName for RS operator in explain formatted
Date Thu, 20 Apr 2017 00:36:04 GMT

    [ https://issues.apache.org/jira/browse/HIVE-16485?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15975811#comment-15975811
] 

Pengcheng Xiong edited comment on HIVE-16485 at 4/20/17 12:35 AM:
------------------------------------------------------------------

sample output
for
{code}
explain formatted
SELECT x.key, z.value, y.value
FROM srcTable x JOIN srcTable y ON (x.key = y.key) 
JOIN srcTable z ON (x.value = z.value)
{code}
{code}
{"STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE
PLANS":{"Stage-1":{"Tez":{"DagId:":"pxiong_20170419172827_bf3a57c0-fa55-437f-8194-49a97b95c4aa:33","Edges:":{"Reducer
2":[{"parent":"Map 1","type":"SIMPLE_EDGE"},{"parent":"Map 4","type":"SIMPLE_EDGE"}],"Reducer
3":[{"parent":"Map 5","type":"SIMPLE_EDGE"},{"parent":"Reducer 2","type":"SIMPLE_EDGE"}]},"DagName:":"","Vertices:":{"Map
1":{"Map Operator Tree:":[{"TableScan":{"alias:":"x","Statistics:":"Num rows: 1 Data size:
0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"TS_0","children":{"Filter Operator":{"predicate:":"(key
is not null and value is not null) (type: boolean)","Statistics:":"Num rows: 1 Data size:
0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"FIL_22","children":{"Select Operator":{"expressions:":"key
(type: string), value (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num
rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"SEL_2","children":{"Reduce
Output Operator":{"key expressions:":"_col0 (type: string)","sort order:":"+","Map-reduce
partition columns:":"_col0 (type: string)","Statistics:":"Num rows: 1 Data size: 0 Basic stats:
PARTIAL Column stats: NONE","value expressions:":"_col1 (type: string)","OperatorId:":"RS_9","outputname:":"Reducer
2"}}}}}}}}],"Execution mode:":"llap","LLAP IO:":"no inputs"},"Map 4":{"Map Operator Tree:":[{"TableScan":{"alias:":"y","Statistics:":"Num
rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"TS_3","children":{"Filter
Operator":{"predicate:":"key is not null (type: boolean)","Statistics:":"Num rows: 1 Data
size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"FIL_23","children":{"Select
Operator":{"expressions:":"key (type: string), value (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num
rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"SEL_5","children":{"Reduce
Output Operator":{"key expressions:":"_col0 (type: string)","sort order:":"+","Map-reduce
partition columns:":"_col0 (type: string)","Statistics:":"Num rows: 1 Data size: 0 Basic stats:
PARTIAL Column stats: NONE","value expressions:":"_col1 (type: string)","OperatorId:":"RS_10","outputname:":"Reducer
2"}}}}}}}}],"Execution mode:":"llap","LLAP IO:":"no inputs"},"Map 5":{"Map Operator Tree:":[{"TableScan":{"alias:":"z","Statistics:":"Num
rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"TS_6","children":{"Filter
Operator":{"predicate:":"value is not null (type: boolean)","Statistics:":"Num rows: 1 Data
size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"FIL_24","children":{"Select
Operator":{"expressions:":"value (type: string)","outputColumnNames:":["_col0"],"Statistics:":"Num
rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"SEL_8","children":{"Reduce
Output Operator":{"key expressions:":"_col0 (type: string)","sort order:":"+","Map-reduce
partition columns:":"_col0 (type: string)","Statistics:":"Num rows: 1 Data size: 0 Basic stats:
PARTIAL Column stats: NONE","OperatorId:":"RS_13","outputname:":"Reducer 3"}}}}}}}}],"Execution
mode:":"llap","LLAP IO:":"no inputs"},"Reducer 2":{"Execution mode:":"llap","Reduce Operator
Tree:":{"Merge Join Operator":{"condition map:":[{"":"Inner Join 0 to 1"}],"keys:":{"0":"_col0
(type: string)","1":"_col0 (type: string)"},"outputColumnNames:":["_col0","_col1","_col3"],"Statistics:":"Num
rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"MERGEJOIN_25","children":{"Reduce
Output Operator":{"key expressions:":"_col1 (type: string)","sort order:":"+","Map-reduce
partition columns:":"_col1 (type: string)","Statistics:":"Num rows: 1 Data size: 0 Basic stats:
PARTIAL Column stats: NONE","value expressions:":"_col0 (type: string), _col3 (type: string)","OperatorId:":"RS_12","outputname:":"Reducer
3"}}}}},"Reducer 3":{"Execution mode:":"llap","Reduce Operator Tree:":{"Merge Join Operator":{"condition
map:":[{"":"Inner Join 0 to 1"}],"keys:":{"0":"_col1 (type: string)","1":"_col0 (type: string)"},"outputColumnNames:":["_col0","_col3","_col4"],"Statistics:":"Num
rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"MERGEJOIN_26","children":{"Select
Operator":{"expressions:":"_col0 (type: string), _col4 (type: string), _col3 (type: string)","outputColumnNames:":["_col0","_col1","_col2"],"Statistics:":"Num
rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"SEL_15","children":{"File
Output Operator":{"compressed:":"false","Statistics:":"Num rows: 1 Data size: 0 Basic stats:
PARTIAL Column stats: NONE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output
format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_16"}}}}}}}}}},"Stage-0":{"Fetch
Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_27"}}}}}}
{code}

enable vectorization
{code}
{"STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE
PLANS":{"Stage-1":{"Tez":{"DagId:":"pxiong_20170419172828_b99dbd19-74d2-449c-a2b3-c7a900de6dd4:35","Edges:":{"Reducer
2":[{"parent":"Map 1","type":"SIMPLE_EDGE"},{"parent":"Map 4","type":"SIMPLE_EDGE"}],"Reducer
3":[{"parent":"Map 5","type":"SIMPLE_EDGE"},{"parent":"Reducer 2","type":"SIMPLE_EDGE"}]},"DagName:":"","Vertices:":{"Map
1":{"Map Operator Tree:":[{"TableScan":{"alias:":"x","Statistics:":"Num rows: 1 Data size:
0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"TS_0","children":{"Filter Operator":{"predicate:":"(key
is not null and value is not null) (type: boolean)","Statistics:":"Num rows: 1 Data size:
0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"FIL_27","children":{"Select Operator":{"expressions:":"key
(type: string), value (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num
rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"SEL_28","children":{"Reduce
Output Operator":{"key expressions:":"_col0 (type: string)","sort order:":"+","Map-reduce
partition columns:":"_col0 (type: string)","Statistics:":"Num rows: 1 Data size: 0 Basic stats:
PARTIAL Column stats: NONE","value expressions:":"_col1 (type: string)","OperatorId:":"RS_29"}}}}}}}}],"Execution
mode:":"vectorized, llap","LLAP IO:":"no inputs"},"Map 4":{"Map Operator Tree:":[{"TableScan":{"alias:":"y","Statistics:":"Num
rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"TS_3","children":{"Filter
Operator":{"predicate:":"key is not null (type: boolean)","Statistics:":"Num rows: 1 Data
size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"FIL_30","children":{"Select
Operator":{"expressions:":"key (type: string), value (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num
rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"SEL_31","children":{"Reduce
Output Operator":{"key expressions:":"_col0 (type: string)","sort order:":"+","Map-reduce
partition columns:":"_col0 (type: string)","Statistics:":"Num rows: 1 Data size: 0 Basic stats:
PARTIAL Column stats: NONE","value expressions:":"_col1 (type: string)","OperatorId:":"RS_32"}}}}}}}}],"Execution
mode:":"vectorized, llap","LLAP IO:":"no inputs"},"Map 5":{"Map Operator Tree:":[{"TableScan":{"alias:":"z","Statistics:":"Num
rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"TS_6","children":{"Filter
Operator":{"predicate:":"value is not null (type: boolean)","Statistics:":"Num rows: 1 Data
size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"FIL_33","children":{"Select
Operator":{"expressions:":"value (type: string)","outputColumnNames:":["_col0"],"Statistics:":"Num
rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"SEL_34","children":{"Reduce
Output Operator":{"key expressions:":"_col0 (type: string)","sort order:":"+","Map-reduce
partition columns:":"_col0 (type: string)","Statistics:":"Num rows: 1 Data size: 0 Basic stats:
PARTIAL Column stats: NONE","OperatorId:":"RS_35"}}}}}}}}],"Execution mode:":"vectorized,
llap","LLAP IO:":"no inputs"},"Reducer 2":{"Execution mode:":"llap","Reduce Operator Tree:":{"Merge
Join Operator":{"condition map:":[{"":"Inner Join 0 to 1"}],"keys:":{"0":"_col0 (type: string)","1":"_col0
(type: string)"},"outputColumnNames:":["_col0","_col1","_col3"],"Statistics:":"Num rows: 1
Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"MERGEJOIN_25","children":{"Reduce
Output Operator":{"key expressions:":"_col1 (type: string)","sort order:":"+","Map-reduce
partition columns:":"_col1 (type: string)","Statistics:":"Num rows: 1 Data size: 0 Basic stats:
PARTIAL Column stats: NONE","value expressions:":"_col0 (type: string), _col3 (type: string)","OperatorId:":"RS_12","outputname:":"Reducer
3"}}}}},"Reducer 3":{"Execution mode:":"llap","Reduce Operator Tree:":{"Merge Join Operator":{"condition
map:":[{"":"Inner Join 0 to 1"}],"keys:":{"0":"_col1 (type: string)","1":"_col0 (type: string)"},"outputColumnNames:":["_col0","_col3","_col4"],"Statistics:":"Num
rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"MERGEJOIN_26","children":{"Select
Operator":{"expressions:":"_col0 (type: string), _col4 (type: string), _col3 (type: string)","outputColumnNames:":["_col0","_col1","_col2"],"Statistics:":"Num
rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"SEL_15","children":{"File
Output Operator":{"compressed:":"false","Statistics:":"Num rows: 1 Data size: 0 Basic stats:
PARTIAL Column stats: NONE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output
format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_16"}}}}}}}}}},"Stage-0":{"Fetch
Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_36"}}}}}}
{code}


was (Author: pxiong):
sample output
for
{code}
explain formatted
SELECT x.key, z.value, y.value
FROM srcTable x JOIN srcTable y ON (x.key = y.key) 
JOIN srcTable z ON (x.value = z.value)
{code}
{code}
{"STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE
PLANS":{"Stage-1":{"Tez":{"DagId:":"pxiong_20170419172827_bf3a57c0-fa55-437f-8194-49a97b95c4aa:33","Edges:":{"Reducer
2":[{"parent":"Map 1","type":"SIMPLE_EDGE"},{"parent":"Map 4","type":"SIMPLE_EDGE"}],"Reducer
3":[{"parent":"Map 5","type":"SIMPLE_EDGE"},{"parent":"Reducer 2","type":"SIMPLE_EDGE"}]},"DagName:":"","Vertices:":{"Map
1":{"Map Operator Tree:":[{"TableScan":{"alias:":"x","Statistics:":"Num rows: 1 Data size:
0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"TS_0","children":{"Filter Operator":{"predicate:":"(key
is not null and value is not null) (type: boolean)","Statistics:":"Num rows: 1 Data size:
0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"FIL_22","children":{"Select Operator":{"expressions:":"key
(type: string), value (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num
rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"SEL_2","children":{"Reduce
Output Operator":{"key expressions:":"_col0 (type: string)","sort order:":"+","Map-reduce
partition columns:":"_col0 (type: string)","Statistics:":"Num rows: 1 Data size: 0 Basic stats:
PARTIAL Column stats: NONE","value expressions:":"_col1 (type: string)","OperatorId:":"RS_9","outputname:":"Reducer
2"}}}}}}}}],"Execution mode:":"llap","LLAP IO:":"no inputs"},"Map 4":{"Map Operator Tree:":[{"TableScan":{"alias:":"y","Statistics:":"Num
rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"TS_3","children":{"Filter
Operator":{"predicate:":"key is not null (type: boolean)","Statistics:":"Num rows: 1 Data
size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"FIL_23","children":{"Select
Operator":{"expressions:":"key (type: string), value (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num
rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"SEL_5","children":{"Reduce
Output Operator":{"key expressions:":"_col0 (type: string)","sort order:":"+","Map-reduce
partition columns:":"_col0 (type: string)","Statistics:":"Num rows: 1 Data size: 0 Basic stats:
PARTIAL Column stats: NONE","value expressions:":"_col1 (type: string)","OperatorId:":"RS_10","outputname:":"Reducer
2"}}}}}}}}],"Execution mode:":"llap","LLAP IO:":"no inputs"},"Map 5":{"Map Operator Tree:":[{"TableScan":{"alias:":"z","Statistics:":"Num
rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"TS_6","children":{"Filter
Operator":{"predicate:":"value is not null (type: boolean)","Statistics:":"Num rows: 1 Data
size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"FIL_24","children":{"Select
Operator":{"expressions:":"value (type: string)","outputColumnNames:":["_col0"],"Statistics:":"Num
rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"SEL_8","children":{"Reduce
Output Operator":{"key expressions:":"_col0 (type: string)","sort order:":"+","Map-reduce
partition columns:":"_col0 (type: string)","Statistics:":"Num rows: 1 Data size: 0 Basic stats:
PARTIAL Column stats: NONE","OperatorId:":"RS_13","outputname:":"Reducer 3"}}}}}}}}],"Execution
mode:":"llap","LLAP IO:":"no inputs"},"Reducer 2":{"Execution mode:":"llap","Reduce Operator
Tree:":{"Merge Join Operator":{"condition map:":[{"":"Inner Join 0 to 1"}],"keys:":{"0":"_col0
(type: string)","1":"_col0 (type: string)"},"outputColumnNames:":["_col0","_col1","_col3"],"Statistics:":"Num
rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"MERGEJOIN_25","children":{"Reduce
Output Operator":{"key expressions:":"_col1 (type: string)","sort order:":"+","Map-reduce
partition columns:":"_col1 (type: string)","Statistics:":"Num rows: 1 Data size: 0 Basic stats:
PARTIAL Column stats: NONE","value expressions:":"_col0 (type: string), _col3 (type: string)","OperatorId:":"RS_12","outputname:":"Reducer
3"}}}}},"Reducer 3":{"Execution mode:":"llap","Reduce Operator Tree:":{"Merge Join Operator":{"condition
map:":[{"":"Inner Join 0 to 1"}],"keys:":{"0":"_col1 (type: string)","1":"_col0 (type: string)"},"outputColumnNames:":["_col0","_col3","_col4"],"Statistics:":"Num
rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"MERGEJOIN_26","children":{"Select
Operator":{"expressions:":"_col0 (type: string), _col4 (type: string), _col3 (type: string)","outputColumnNames:":["_col0","_col1","_col2"],"Statistics:":"Num
rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"SEL_15","children":{"File
Output Operator":{"compressed:":"false","Statistics:":"Num rows: 1 Data size: 0 Basic stats:
PARTIAL Column stats: NONE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output
format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_16"}}}}}}}}}},"Stage-0":{"Fetch
Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_27"}}}}}}
{code}

> Enable outputName for RS operator in explain formatted
> ------------------------------------------------------
>
>                 Key: HIVE-16485
>                 URL: https://issues.apache.org/jira/browse/HIVE-16485
>             Project: Hive
>          Issue Type: Bug
>            Reporter: Pengcheng Xiong
>            Assignee: Pengcheng Xiong
>




--
This message was sent by Atlassian JIRA
(v6.3.15#6346)

Mime
View raw message