hive-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Zoltan Haindrich (JIRA)" <j...@apache.org>
Subject [jira] [Updated] (HIVE-18235) Columnstats gather on mm tables: re-enable disabled test
Date Mon, 12 Feb 2018 10:24:00 GMT

     [ https://issues.apache.org/jira/browse/HIVE-18235?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]

Zoltan Haindrich updated HIVE-18235:
------------------------------------
        Summary: Columnstats gather on mm tables: re-enable disabled test  (was: Columnstats
gather fails for insert_only table)
    Description: 
test: dp_counter_mm.q

at:
{code}
insert overwrite table src2 partition (value) select * from src where key < 100;
{code}

produces:
{code}
2017-12-06T02:39:54,447 DEBUG [d709e6e0-7573-4c79-bb38-b043a88a8dde main] metrics.PerfLogger:
</PERFLOG method=set_aggr_stats_for start=1512556794433 end=1512556794447 duration=14 from=org.apache.hadoop.hive.metastore.RetryingHMSHandler
threadId=0 retryCount=-1 error=true>
2017-12-06T02:39:54,447 DEBUG [d709e6e0-7573-4c79-bb38-b043a88a8dde main] metadata.Hive: NoSuchObjectException(message:Partition
for which stats is gathered doesn't exist.)
        at org.apache.hadoop.hive.metastore.ObjectStore.updatePartitionColumnStatistics(ObjectStore.java:7644)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at org.apache.hadoop.hive.metastore.RawStoreProxy.invoke(RawStoreProxy.java:97)
        at com.sun.proxy.$Proxy52.updatePartitionColumnStatistics(Unknown Source)
        at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.updatePartitonColStats(HiveMetaStore.java:5340)
        at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.set_aggr_stats_for(HiveMetaStore.java:6853)
        at sun.reflect.GeneratedMethodAccessor81.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at org.apache.hadoop.hive.metastore.RetryingHMSHandler.invokeInternal(RetryingHMSHandler.java:147)
        at org.apache.hadoop.hive.metastore.RetryingHMSHandler.invoke(RetryingHMSHandler.java:108)
        at com.sun.proxy.$Proxy54.set_aggr_stats_for(Unknown Source)
        at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.setPartitionColumnStatistics(HiveMetaStoreClient.java:1748)
        at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.setPartitionColumnStatistics(SessionHiveMetaStoreClient.java:374)
        at sun.reflect.GeneratedMethodAccessor80.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.invoke(RetryingMetaStoreClient.java:211)
        at com.sun.proxy.$Proxy55.setPartitionColumnStatistics(Unknown Source)
        at org.apache.hadoop.hive.ql.metadata.Hive.setPartitionColumnStatistics(Hive.java:4215)
        at org.apache.hadoop.hive.ql.stats.ColStatsProcessor.persistColumnStats(ColStatsProcessor.java:180)
        at org.apache.hadoop.hive.ql.stats.ColStatsProcessor.process(ColStatsProcessor.java:84)
        at org.apache.hadoop.hive.ql.exec.StatsTask.execute(StatsTask.java:108)
        at org.apache.hadoop.hive.ql.exec.Task.executeTask(Task.java:205)
        at org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential(TaskRunner.java:97)
        at org.apache.hadoop.hive.ql.Driver.launchTask(Driver.java:2230)
        at org.apache.hadoop.hive.ql.Driver.execute(Driver.java:1882)
        at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:1613)
        at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1358)
        at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1346)
        at org.apache.hadoop.hive.cli.CliDriver.processLocalCmd(CliDriver.java:239)
{code}

  was:

test: dp_counter_mm.q

at:
{code}
insert overwrite table src2 partition (value) select * from src where key < 100;
{code}

produces:
{code}
2017-12-06T02:39:54,447 DEBUG [d709e6e0-7573-4c79-bb38-b043a88a8dde main] metrics.PerfLogger:
</PERFLOG method=set_aggr_stats_for start=1512556794433 end=1512556794447 duration=14 from=org.apache.hadoop.hive.metastore.RetryingHMSHandler
threadId=0 retryCount=-1 error=true>
2017-12-06T02:39:54,447 DEBUG [d709e6e0-7573-4c79-bb38-b043a88a8dde main] metadata.Hive: NoSuchObjectException(message:Partition
for which stats is gathered doesn't exist.)
        at org.apache.hadoop.hive.metastore.ObjectStore.updatePartitionColumnStatistics(ObjectStore.java:7644)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at org.apache.hadoop.hive.metastore.RawStoreProxy.invoke(RawStoreProxy.java:97)
        at com.sun.proxy.$Proxy52.updatePartitionColumnStatistics(Unknown Source)
        at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.updatePartitonColStats(HiveMetaStore.java:5340)
        at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.set_aggr_stats_for(HiveMetaStore.java:6853)
        at sun.reflect.GeneratedMethodAccessor81.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at org.apache.hadoop.hive.metastore.RetryingHMSHandler.invokeInternal(RetryingHMSHandler.java:147)
        at org.apache.hadoop.hive.metastore.RetryingHMSHandler.invoke(RetryingHMSHandler.java:108)
        at com.sun.proxy.$Proxy54.set_aggr_stats_for(Unknown Source)
        at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.setPartitionColumnStatistics(HiveMetaStoreClient.java:1748)
        at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.setPartitionColumnStatistics(SessionHiveMetaStoreClient.java:374)
        at sun.reflect.GeneratedMethodAccessor80.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.invoke(RetryingMetaStoreClient.java:211)
        at com.sun.proxy.$Proxy55.setPartitionColumnStatistics(Unknown Source)
        at org.apache.hadoop.hive.ql.metadata.Hive.setPartitionColumnStatistics(Hive.java:4215)
        at org.apache.hadoop.hive.ql.stats.ColStatsProcessor.persistColumnStats(ColStatsProcessor.java:180)
        at org.apache.hadoop.hive.ql.stats.ColStatsProcessor.process(ColStatsProcessor.java:84)
        at org.apache.hadoop.hive.ql.exec.StatsTask.execute(StatsTask.java:108)
        at org.apache.hadoop.hive.ql.exec.Task.executeTask(Task.java:205)
        at org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential(TaskRunner.java:97)
        at org.apache.hadoop.hive.ql.Driver.launchTask(Driver.java:2230)
        at org.apache.hadoop.hive.ql.Driver.execute(Driver.java:1882)
        at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:1613)
        at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1358)
        at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1346)
        at org.apache.hadoop.hive.cli.CliDriver.processLocalCmd(CliDriver.java:239)
{code}


#1)

* remove the comment/set to disable colstats gather

> Columnstats gather on mm tables: re-enable disabled test
> --------------------------------------------------------
>
>                 Key: HIVE-18235
>                 URL: https://issues.apache.org/jira/browse/HIVE-18235
>             Project: Hive
>          Issue Type: Sub-task
>          Components: Statistics, Transactions
>            Reporter: Zoltan Haindrich
>            Assignee: Zoltan Haindrich
>            Priority: Major
>         Attachments: HIVE-18235.01.patch
>
>
> test: dp_counter_mm.q
> at:
> {code}
> insert overwrite table src2 partition (value) select * from src where key < 100;
> {code}
> produces:
> {code}
> 2017-12-06T02:39:54,447 DEBUG [d709e6e0-7573-4c79-bb38-b043a88a8dde main] metrics.PerfLogger:
</PERFLOG method=set_aggr_stats_for start=1512556794433 end=1512556794447 duration=14 from=org.apache.hadoop.hive.metastore.RetryingHMSHandler
threadId=0 retryCount=-1 error=true>
> 2017-12-06T02:39:54,447 DEBUG [d709e6e0-7573-4c79-bb38-b043a88a8dde main] metadata.Hive:
NoSuchObjectException(message:Partition for which stats is gathered doesn't exist.)
>         at org.apache.hadoop.hive.metastore.ObjectStore.updatePartitionColumnStatistics(ObjectStore.java:7644)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:498)
>         at org.apache.hadoop.hive.metastore.RawStoreProxy.invoke(RawStoreProxy.java:97)
>         at com.sun.proxy.$Proxy52.updatePartitionColumnStatistics(Unknown Source)
>         at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.updatePartitonColStats(HiveMetaStore.java:5340)
>         at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.set_aggr_stats_for(HiveMetaStore.java:6853)
>         at sun.reflect.GeneratedMethodAccessor81.invoke(Unknown Source)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:498)
>         at org.apache.hadoop.hive.metastore.RetryingHMSHandler.invokeInternal(RetryingHMSHandler.java:147)
>         at org.apache.hadoop.hive.metastore.RetryingHMSHandler.invoke(RetryingHMSHandler.java:108)
>         at com.sun.proxy.$Proxy54.set_aggr_stats_for(Unknown Source)
>         at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.setPartitionColumnStatistics(HiveMetaStoreClient.java:1748)
>         at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.setPartitionColumnStatistics(SessionHiveMetaStoreClient.java:374)
>         at sun.reflect.GeneratedMethodAccessor80.invoke(Unknown Source)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:498)
>         at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.invoke(RetryingMetaStoreClient.java:211)
>         at com.sun.proxy.$Proxy55.setPartitionColumnStatistics(Unknown Source)
>         at org.apache.hadoop.hive.ql.metadata.Hive.setPartitionColumnStatistics(Hive.java:4215)
>         at org.apache.hadoop.hive.ql.stats.ColStatsProcessor.persistColumnStats(ColStatsProcessor.java:180)
>         at org.apache.hadoop.hive.ql.stats.ColStatsProcessor.process(ColStatsProcessor.java:84)
>         at org.apache.hadoop.hive.ql.exec.StatsTask.execute(StatsTask.java:108)
>         at org.apache.hadoop.hive.ql.exec.Task.executeTask(Task.java:205)
>         at org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential(TaskRunner.java:97)
>         at org.apache.hadoop.hive.ql.Driver.launchTask(Driver.java:2230)
>         at org.apache.hadoop.hive.ql.Driver.execute(Driver.java:1882)
>         at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:1613)
>         at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1358)
>         at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1346)
>         at org.apache.hadoop.hive.cli.CliDriver.processLocalCmd(CliDriver.java:239)
> {code}



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

Mime
View raw message