hive-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Gopal V (JIRA)" <j...@apache.org>
Subject [jira] [Created] (HIVE-12169) PCR: Should not fetch all the Partitions to Client always
Date Wed, 14 Oct 2015 02:03:05 GMT
Gopal V created HIVE-12169:
------------------------------

             Summary: PCR: Should not fetch all the Partitions to Client always
                 Key: HIVE-12169
                 URL: https://issues.apache.org/jira/browse/HIVE-12169
             Project: Hive
          Issue Type: Bug
          Components: Physical Optimizer
    Affects Versions: 1.3.0, 2.0.0
            Reporter: Gopal V
            Assignee: Gopal V


Even for simple queries which only have a column filter, the PCR does not check if PPR has
run before & therefore pulls all partition columns down for queries which do not have
a partition filter at all.

{code}
org.apache.hadoop.hive.metastore.MetaStoreDirectSql.ensureList(Object) MetaStoreDirectSql.java:1656
org.apache.hadoop.hive.metastore.MetaStoreDirectSql.loopJoinOrderedResult(TreeMap, String,
int, MetaStoreDirectSql$ApplyFunc) MetaStoreDirectSql.java:896
org.apache.hadoop.hive.metastore.MetaStoreDirectSql.getPartitionsFromPartitionIds(String,
String, Boolean, List) MetaStoreDirectSql.java:644
org.apache.hadoop.hive.metastore.MetaStoreDirectSql.getPartitionsViaSqlFilterInternal(String,
String, Boolean, String, List, List, Integer) MetaStoreDirectSql.java:511
org.apache.hadoop.hive.metastore.MetaStoreDirectSql.getPartitionsViaSqlFilter(String, String,
List) MetaStoreDirectSql.java:376
org.apache.hadoop.hive.metastore.ObjectStore$5.getSqlResult(ObjectStore$GetHelper) ObjectStore.java:2159
org.apache.hadoop.hive.metastore.ObjectStore$5.getSqlResult(ObjectStore$GetHelper) ObjectStore.java:2146
org.apache.hadoop.hive.metastore.ObjectStore$GetHelper.run(boolean) ObjectStore.java:2392
org.apache.hadoop.hive.metastore.ObjectStore.getPartitionsByExprInternal(String, String, byte[],
String, short, List, boolean, boolean) ObjectStore.java:2146
org.apache.hadoop.hive.metastore.ObjectStore.getPartitionsByExpr(String, String, byte[], String,
short, List) ObjectStore.java:2136
sun.reflect.GeneratedMethodAccessor72.invoke(Object, Object[])
sun.reflect.DelegatingMethodAccessorImpl.invoke(Object, Object[]) DelegatingMethodAccessorImpl.java:43
java.lang.reflect.Method.invoke(Object, Object[]) Method.java:497
org.apache.hadoop.hive.metastore.RawStoreProxy.invoke(Object, Method, Object[]) RawStoreProxy.java:117
com.sun.proxy.$Proxy28.getPartitionsByExpr(String, String, byte[], String, short, List)
org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.get_partitions_by_expr(PartitionsByExprRequest)
HiveMetaStore.java:4545
sun.reflect.GeneratedMethodAccessor71.invoke(Object, Object[])
sun.reflect.DelegatingMethodAccessorImpl.invoke(Object, Object[]) DelegatingMethodAccessorImpl.java:43
java.lang.reflect.Method.invoke(Object, Object[]) Method.java:497
org.apache.hadoop.hive.metastore.RetryingHMSHandler.invokeInternal(Object, Method, Object[])
RetryingHMSHandler.java:138
org.apache.hadoop.hive.metastore.RetryingHMSHandler.invoke(Object, Method, Object[]) RetryingHMSHandler.java:99
com.sun.proxy.$Proxy30.get_partitions_by_expr(PartitionsByExprRequest)
org.apache.hadoop.hive.metastore.HiveMetaStoreClient.listPartitionsByExpr(String, String,
byte[], String, short, List) HiveMetaStoreClient.java:1160
sun.reflect.GeneratedMethodAccessor70.invoke(Object, Object[])
sun.reflect.DelegatingMethodAccessorImpl.invoke(Object, Object[]) DelegatingMethodAccessorImpl.java:43
java.lang.reflect.Method.invoke(Object, Object[]) Method.java:497
org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.invoke(Object, Method, Object[])
RetryingMetaStoreClient.java:156
com.sun.proxy.$Proxy31.listPartitionsByExpr(String, String, byte[], String, short, List)
org.apache.hadoop.hive.ql.metadata.Hive.getPartitionsByExpr(Table, ExprNodeGenericFuncDesc,
HiveConf, List) Hive.java:2361
org.apache.hadoop.hive.ql.optimizer.ppr.PartitionPruner.getPartitionsFromServer(Table, ExprNodeGenericFuncDesc,
HiveConf, String, Set, boolean) PartitionPruner.java:420
org.apache.hadoop.hive.ql.optimizer.ppr.PartitionPruner.prune(Table, ExprNodeDesc, HiveConf,
String, Map) PartitionPruner.java:221
org.apache.hadoop.hive.ql.optimizer.ppr.PartitionPruner.prune(TableScanOperator, ParseContext,
String) PartitionPruner.java:144
org.apache.hadoop.hive.ql.parse.ParseContext.getPrunedPartitions(String, TableScanOperator)
ParseContext.java:460
org.apache.hadoop.hive.ql.optimizer.pcr.PcrOpProcFactory$FilterPCR.process(Node, Stack, NodeProcessorCtx,
Object[]) PcrOpProcFactory.java:110
org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher.dispatch(Node, Stack, Object[]) DefaultRuleDispatcher.java:90
org.apache.hadoop.hive.ql.lib.DefaultGraphWalker.dispatchAndReturn(Node, Stack) DefaultGraphWalker.java:105
org.apache.hadoop.hive.ql.lib.DefaultGraphWalker.dispatch(Node, Stack) DefaultGraphWalker.java:89
org.apache.hadoop.hive.ql.lib.DefaultGraphWalker.walk(Node) DefaultGraphWalker.java:158
org.apache.hadoop.hive.ql.lib.DefaultGraphWalker.startWalking(Collection, HashMap) DefaultGraphWalker.java:120
org.apache.hadoop.hive.ql.optimizer.pcr.PartitionConditionRemover.transform(ParseContext)
PartitionConditionRemover.java:86
org.apache.hadoop.hive.ql.optimizer.Optimizer.optimize() Optimizer.java:209
org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.analyzeInternal(ASTNode, SemanticAnalyzer$PlannerContext)
SemanticAnalyzer.java:10195
org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.analyzeInternal(ASTNode) SemanticAnalyzer.java:9980
org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(ASTNode, Context) BaseSemanticAnalyzer.java:240
org.apache.hadoop.hive.ql.Driver.compile(String, boolean) Driver.java:440
org.apache.hadoop.hive.ql.Driver.compile(String) Driver.java:311
org.apache.hadoop.hive.ql.Driver.compileInternal(String) Driver.java:1162
org.apache.hadoop.hive.ql.Driver.runInternal(String, boolean) Driver.java:1215
org.apache.hadoop.hive.ql.Driver.run(String, boolean) Driver.java:1091
org.apache.hadoop.hive.ql.Driver.run(String) Driver.java:1081
org.apache.hadoop.hive.cli.CliDriver.processLocalCmd(String, CommandProcessor, CliSessionState)
CliDriver.java:225
org.apache.hadoop.hive.cli.CliDriver.processCmd(String) CliDriver.java:177
org.apache.hadoop.hive.cli.CliDriver.processLine(String, boolean) CliDriver.java:388
org.apache.hadoop.hive.cli.CliDriver.executeDriver(CliSessionState, HiveConf, OptionsProcessor)
CliDriver.java:759
org.apache.hadoop.hive.cli.CliDriver.run(String[]) CliDriver.java:704
org.apache.hadoop.hive.cli.CliDriver.main(String[]) CliDriver.java:633
sun.reflect.NativeMethodAccessorImpl.invoke0(Method, Object, Object[]) NativeMethodAccessorImpl.java
(native)
sun.reflect.NativeMethodAccessorImpl.invoke(Object, Object[]) NativeMethodAccessorImpl.java:62
sun.reflect.DelegatingMethodAccessorImpl.invoke(Object, Object[]) DelegatingMethodAccessorImpl.java:43
java.lang.reflect.Method.invoke(Object, Object[]) Method.java:497
org.apache.hadoop.util.RunJar.run(String[]) RunJar.java:221
org.apache.hadoop.util.RunJar.main(String[]) RunJar.java:136
{code}



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

Mime
View raw message