hive-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Arkadiusz Gasior (JIRA)" <j...@apache.org>
Subject [jira] [Updated] (HIVE-7657) Map of union of 3 or more types cannot be joined
Date Wed, 13 Aug 2014 15:32:13 GMT

     [ https://issues.apache.org/jira/browse/HIVE-7657?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]

Arkadiusz Gasior updated HIVE-7657:
-----------------------------------

    Description: 
Joining table which contains map of union of 3 types or more is causing serialization issues.

Given tables:
ag_map_of_uniontype:
# col_name data_type comment
hash string None
custom_fields map<string,uniontype<bigint,double,string,boolean,int,array<uniontype<bigint,double,string,boolean,int>>,map<string,uniontype<bigint,double,string,boolean,int>>>>
None

ag_simple_join_table:
# col_name data_type comment
hash string None
size int None

Query will throw java.lang.RuntimeException: 
{code}
select
i.hash,
i.custom_fields,
b.size
from ag_map_of_uniontype i LEFT OUTER JOIN
ag_simple_join_table b ON (i.hash = b.hash);
{code}

{code}
org.apache.hadoop.hive.ql.exec.MapJoinOperator: Unxpected exception: java.util.ArrayList cannot
be cast to org.apache.hadoop.hive.serde2.objectinspector.UnionObject
java.lang.ClassCastException: java.util.ArrayList cannot be cast to org.apache.hadoop.hive.serde2.objectinspector.UnionObject
	at org.apache.hadoop.hive.serde2.objectinspector.StandardUnionObjectInspector.getTag(StandardUnionObjectInspector.java:91)
	at org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.serialize(LazySimpleSerDe.java:559)
	at org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.serialize(LazySimpleSerDe.java:529)
	at org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.serializeField(LazySimpleSerDe.java:439)
	at org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.serialize(LazySimpleSerDe.java:423)
	at org.apache.hadoop.hive.ql.exec.FileSinkOperator.processOp(FileSinkOperator.java:618)
	at org.apache.hadoop.hive.ql.exec.Operator.process(Operator.java:504)
	at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:847)
	at org.apache.hadoop.hive.ql.exec.SelectOperator.processOp(SelectOperator.java:87)
	at org.apache.hadoop.hive.ql.exec.Operator.process(Operator.java:504)
	at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:847)
	at org.apache.hadoop.hive.ql.exec.CommonJoinOperator.genAllOneUniqueJoinObject(CommonJoinOperator.java:671)
	at org.apache.hadoop.hive.ql.exec.CommonJoinOperator.checkAndGenObject(CommonJoinOperator.java:754)
	at org.apache.hadoop.hive.ql.exec.MapJoinOperator.processOp(MapJoinOperator.java:222)
	at org.apache.hadoop.hive.ql.exec.Operator.process(Operator.java:504)
	at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:847)
	at org.apache.hadoop.hive.ql.exec.TableScanOperator.processOp(TableScanOperator.java:91)
	at org.apache.hadoop.hive.ql.exec.Operator.process(Operator.java:504)
	at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:847)
	at org.apache.hadoop.hive.ql.exec.MapOperator.process(MapOperator.java:519)
	at org.apache.hadoop.hive.ql.exec.mr.ExecMapper.map(ExecMapper.java:157)
	at org.apache.hadoop.mapred.MapRunner.run(MapRunner.java:50)
	at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:417)
	at org.apache.hadoop.mapred.MapTask.run(MapTask.java:332)
	at org.apache.hadoop.mapred.Child$4.run(Child.java:268)
	at java.security.AccessController.doPrivileged(Native Method)
	at javax.security.auth.Subject.doAs(Subject.java:415)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1554)
	at org.apache.hadoop.mapred.Child.main(Child.java:262)

org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing row
{"hash":"nzM2VsRBJReFrv4fGeSX","custom_fields":{"listOfIntegers":{5:[{4:10},{4:28},{4:42},{4:51},{4:63},{4:92},{4:113},{4:126},{4:128},{4:130},{4:144},{4:145},{4:146},{4:179},{4:182},{4:198},{4:225},{4:260},{4:232},{4:233},{4:237},{4:238},{4:255}]}}}

{code}

  was:
Handling nullable map of union of 3 types or more is causing serialization issues.

Given tables:
ag_map_of_uniontype:
# col_name data_type comment
hash string None
custom_fields map<string,uniontype<bigint,double,string,boolean,int,array<uniontype<bigint,double,string,boolean,int>>,map<string,uniontype<bigint,double,string,boolean,int>>>>
None

ag_simple_join_table:
# col_name data_type comment
hash string None
size int None

Query will throw java.lang.RuntimeException: 
{code}
select
i.hash,
i.custom_fields,
b.size
from ag_map_of_uniontype i LEFT OUTER JOIN
ag_simple_join_table b ON (i.hash = b.hash);
{code}

{code}
org.apache.hadoop.hive.ql.exec.MapJoinOperator: Unxpected exception: java.util.ArrayList cannot
be cast to org.apache.hadoop.hive.serde2.objectinspector.UnionObject
java.lang.ClassCastException: java.util.ArrayList cannot be cast to org.apache.hadoop.hive.serde2.objectinspector.UnionObject
	at org.apache.hadoop.hive.serde2.objectinspector.StandardUnionObjectInspector.getTag(StandardUnionObjectInspector.java:91)
	at org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.serialize(LazySimpleSerDe.java:559)
	at org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.serialize(LazySimpleSerDe.java:529)
	at org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.serializeField(LazySimpleSerDe.java:439)
	at org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.serialize(LazySimpleSerDe.java:423)
	at org.apache.hadoop.hive.ql.exec.FileSinkOperator.processOp(FileSinkOperator.java:618)
	at org.apache.hadoop.hive.ql.exec.Operator.process(Operator.java:504)
	at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:847)
	at org.apache.hadoop.hive.ql.exec.SelectOperator.processOp(SelectOperator.java:87)
	at org.apache.hadoop.hive.ql.exec.Operator.process(Operator.java:504)
	at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:847)
	at org.apache.hadoop.hive.ql.exec.CommonJoinOperator.genAllOneUniqueJoinObject(CommonJoinOperator.java:671)
	at org.apache.hadoop.hive.ql.exec.CommonJoinOperator.checkAndGenObject(CommonJoinOperator.java:754)
	at org.apache.hadoop.hive.ql.exec.MapJoinOperator.processOp(MapJoinOperator.java:222)
	at org.apache.hadoop.hive.ql.exec.Operator.process(Operator.java:504)
	at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:847)
	at org.apache.hadoop.hive.ql.exec.TableScanOperator.processOp(TableScanOperator.java:91)
	at org.apache.hadoop.hive.ql.exec.Operator.process(Operator.java:504)
	at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:847)
	at org.apache.hadoop.hive.ql.exec.MapOperator.process(MapOperator.java:519)
	at org.apache.hadoop.hive.ql.exec.mr.ExecMapper.map(ExecMapper.java:157)
	at org.apache.hadoop.mapred.MapRunner.run(MapRunner.java:50)
	at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:417)
	at org.apache.hadoop.mapred.MapTask.run(MapTask.java:332)
	at org.apache.hadoop.mapred.Child$4.run(Child.java:268)
	at java.security.AccessController.doPrivileged(Native Method)
	at javax.security.auth.Subject.doAs(Subject.java:415)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1554)
	at org.apache.hadoop.mapred.Child.main(Child.java:262)

org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing row
{"hash":"nzM2VsRBJReFrv4fGeSX","custom_fields":{"listOfIntegers":{5:[{4:10},{4:28},{4:42},{4:51},{4:63},{4:92},{4:113},{4:126},{4:128},{4:130},{4:144},{4:145},{4:146},{4:179},{4:182},{4:198},{4:225},{4:260},{4:232},{4:233},{4:237},{4:238},{4:255}]}}}

{code}

        Summary: Map of union of 3 or more types cannot be joined  (was: Nullable map of union
of 3 or more types cannot be joined)

> Map of union of 3 or more types cannot be joined
> ------------------------------------------------
>
>                 Key: HIVE-7657
>                 URL: https://issues.apache.org/jira/browse/HIVE-7657
>             Project: Hive
>          Issue Type: Bug
>          Components: Serializers/Deserializers
>            Reporter: Arkadiusz Gasior
>            Assignee: Ashish Kumar Singh
>              Labels: serde
>
> Joining table which contains map of union of 3 types or more is causing serialization
issues.
> Given tables:
> ag_map_of_uniontype:
> # col_name data_type comment
> hash string None
> custom_fields map<string,uniontype<bigint,double,string,boolean,int,array<uniontype<bigint,double,string,boolean,int>>,map<string,uniontype<bigint,double,string,boolean,int>>>>
None
> ag_simple_join_table:
> # col_name data_type comment
> hash string None
> size int None
> Query will throw java.lang.RuntimeException: 
> {code}
> select
> i.hash,
> i.custom_fields,
> b.size
> from ag_map_of_uniontype i LEFT OUTER JOIN
> ag_simple_join_table b ON (i.hash = b.hash);
> {code}
> {code}
> org.apache.hadoop.hive.ql.exec.MapJoinOperator: Unxpected exception: java.util.ArrayList
cannot be cast to org.apache.hadoop.hive.serde2.objectinspector.UnionObject
> java.lang.ClassCastException: java.util.ArrayList cannot be cast to org.apache.hadoop.hive.serde2.objectinspector.UnionObject
> 	at org.apache.hadoop.hive.serde2.objectinspector.StandardUnionObjectInspector.getTag(StandardUnionObjectInspector.java:91)
> 	at org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.serialize(LazySimpleSerDe.java:559)
> 	at org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.serialize(LazySimpleSerDe.java:529)
> 	at org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.serializeField(LazySimpleSerDe.java:439)
> 	at org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.serialize(LazySimpleSerDe.java:423)
> 	at org.apache.hadoop.hive.ql.exec.FileSinkOperator.processOp(FileSinkOperator.java:618)
> 	at org.apache.hadoop.hive.ql.exec.Operator.process(Operator.java:504)
> 	at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:847)
> 	at org.apache.hadoop.hive.ql.exec.SelectOperator.processOp(SelectOperator.java:87)
> 	at org.apache.hadoop.hive.ql.exec.Operator.process(Operator.java:504)
> 	at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:847)
> 	at org.apache.hadoop.hive.ql.exec.CommonJoinOperator.genAllOneUniqueJoinObject(CommonJoinOperator.java:671)
> 	at org.apache.hadoop.hive.ql.exec.CommonJoinOperator.checkAndGenObject(CommonJoinOperator.java:754)
> 	at org.apache.hadoop.hive.ql.exec.MapJoinOperator.processOp(MapJoinOperator.java:222)
> 	at org.apache.hadoop.hive.ql.exec.Operator.process(Operator.java:504)
> 	at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:847)
> 	at org.apache.hadoop.hive.ql.exec.TableScanOperator.processOp(TableScanOperator.java:91)
> 	at org.apache.hadoop.hive.ql.exec.Operator.process(Operator.java:504)
> 	at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:847)
> 	at org.apache.hadoop.hive.ql.exec.MapOperator.process(MapOperator.java:519)
> 	at org.apache.hadoop.hive.ql.exec.mr.ExecMapper.map(ExecMapper.java:157)
> 	at org.apache.hadoop.mapred.MapRunner.run(MapRunner.java:50)
> 	at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:417)
> 	at org.apache.hadoop.mapred.MapTask.run(MapTask.java:332)
> 	at org.apache.hadoop.mapred.Child$4.run(Child.java:268)
> 	at java.security.AccessController.doPrivileged(Native Method)
> 	at javax.security.auth.Subject.doAs(Subject.java:415)
> 	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1554)
> 	at org.apache.hadoop.mapred.Child.main(Child.java:262)
> org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing
row {"hash":"nzM2VsRBJReFrv4fGeSX","custom_fields":{"listOfIntegers":{5:[{4:10},{4:28},{4:42},{4:51},{4:63},{4:92},{4:113},{4:126},{4:128},{4:130},{4:144},{4:145},{4:146},{4:179},{4:182},{4:198},{4:225},{4:260},{4:232},{4:233},{4:237},{4:238},{4:255}]}}}
> {code}



--
This message was sent by Atlassian JIRA
(v6.2#6252)

Mime
View raw message