Gopal V created HIVE-12169:
------------------------------
Summary: PCR: Should not fetch all the Partitions to Client always
Key: HIVE-12169
URL: https://issues.apache.org/jira/browse/HIVE-12169
Project: Hive
Issue Type: Bug
Components: Physical Optimizer
Affects Versions: 1.3.0, 2.0.0
Reporter: Gopal V
Assignee: Gopal V
Even for simple queries which only have a column filter, the PCR does not check
if PPR has run before & therefore pulls all partition columns down for queries
which do not have a partition filter at all.
{code}
org.apache.hadoop.hive.metastore.MetaStoreDirectSql.ensureList(Object)
MetaStoreDirectSql.java:1656
org.apache.hadoop.hive.metastore.MetaStoreDirectSql.loopJoinOrderedResult(TreeMap,
String, int, MetaStoreDirectSql$ApplyFunc) MetaStoreDirectSql.java:896
org.apache.hadoop.hive.metastore.MetaStoreDirectSql.getPartitionsFromPartitionIds(String,
String, Boolean, List) MetaStoreDirectSql.java:644
org.apache.hadoop.hive.metastore.MetaStoreDirectSql.getPartitionsViaSqlFilterInternal(String,
String, Boolean, String, List, List, Integer) MetaStoreDirectSql.java:511
org.apache.hadoop.hive.metastore.MetaStoreDirectSql.getPartitionsViaSqlFilter(String,
String, List) MetaStoreDirectSql.java:376
org.apache.hadoop.hive.metastore.ObjectStore$5.getSqlResult(ObjectStore$GetHelper)
ObjectStore.java:2159
org.apache.hadoop.hive.metastore.ObjectStore$5.getSqlResult(ObjectStore$GetHelper)
ObjectStore.java:2146
org.apache.hadoop.hive.metastore.ObjectStore$GetHelper.run(boolean)
ObjectStore.java:2392
org.apache.hadoop.hive.metastore.ObjectStore.getPartitionsByExprInternal(String,
String, byte[], String, short, List, boolean, boolean) ObjectStore.java:2146
org.apache.hadoop.hive.metastore.ObjectStore.getPartitionsByExpr(String,
String, byte[], String, short, List) ObjectStore.java:2136
sun.reflect.GeneratedMethodAccessor72.invoke(Object, Object[])
sun.reflect.DelegatingMethodAccessorImpl.invoke(Object, Object[])
DelegatingMethodAccessorImpl.java:43
java.lang.reflect.Method.invoke(Object, Object[]) Method.java:497
org.apache.hadoop.hive.metastore.RawStoreProxy.invoke(Object, Method, Object[])
RawStoreProxy.java:117
com.sun.proxy.$Proxy28.getPartitionsByExpr(String, String, byte[], String,
short, List)
org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.get_partitions_by_expr(PartitionsByExprRequest)
HiveMetaStore.java:4545
sun.reflect.GeneratedMethodAccessor71.invoke(Object, Object[])
sun.reflect.DelegatingMethodAccessorImpl.invoke(Object, Object[])
DelegatingMethodAccessorImpl.java:43
java.lang.reflect.Method.invoke(Object, Object[]) Method.java:497
org.apache.hadoop.hive.metastore.RetryingHMSHandler.invokeInternal(Object,
Method, Object[]) RetryingHMSHandler.java:138
org.apache.hadoop.hive.metastore.RetryingHMSHandler.invoke(Object, Method,
Object[]) RetryingHMSHandler.java:99
com.sun.proxy.$Proxy30.get_partitions_by_expr(PartitionsByExprRequest)
org.apache.hadoop.hive.metastore.HiveMetaStoreClient.listPartitionsByExpr(String,
String, byte[], String, short, List) HiveMetaStoreClient.java:1160
sun.reflect.GeneratedMethodAccessor70.invoke(Object, Object[])
sun.reflect.DelegatingMethodAccessorImpl.invoke(Object, Object[])
DelegatingMethodAccessorImpl.java:43
java.lang.reflect.Method.invoke(Object, Object[]) Method.java:497
org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.invoke(Object, Method,
Object[]) RetryingMetaStoreClient.java:156
com.sun.proxy.$Proxy31.listPartitionsByExpr(String, String, byte[], String,
short, List)
org.apache.hadoop.hive.ql.metadata.Hive.getPartitionsByExpr(Table,
ExprNodeGenericFuncDesc, HiveConf, List) Hive.java:2361
org.apache.hadoop.hive.ql.optimizer.ppr.PartitionPruner.getPartitionsFromServer(Table,
ExprNodeGenericFuncDesc, HiveConf, String, Set, boolean)
PartitionPruner.java:420
org.apache.hadoop.hive.ql.optimizer.ppr.PartitionPruner.prune(Table,
ExprNodeDesc, HiveConf, String, Map) PartitionPruner.java:221
org.apache.hadoop.hive.ql.optimizer.ppr.PartitionPruner.prune(TableScanOperator,
ParseContext, String) PartitionPruner.java:144
org.apache.hadoop.hive.ql.parse.ParseContext.getPrunedPartitions(String,
TableScanOperator) ParseContext.java:460
org.apache.hadoop.hive.ql.optimizer.pcr.PcrOpProcFactory$FilterPCR.process(Node,
Stack, NodeProcessorCtx, Object[]) PcrOpProcFactory.java:110
org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher.dispatch(Node, Stack,
Object[]) DefaultRuleDispatcher.java:90
org.apache.hadoop.hive.ql.lib.DefaultGraphWalker.dispatchAndReturn(Node, Stack)
DefaultGraphWalker.java:105
org.apache.hadoop.hive.ql.lib.DefaultGraphWalker.dispatch(Node, Stack)
DefaultGraphWalker.java:89
org.apache.hadoop.hive.ql.lib.DefaultGraphWalker.walk(Node)
DefaultGraphWalker.java:158
org.apache.hadoop.hive.ql.lib.DefaultGraphWalker.startWalking(Collection,
HashMap) DefaultGraphWalker.java:120
org.apache.hadoop.hive.ql.optimizer.pcr.PartitionConditionRemover.transform(ParseContext)
PartitionConditionRemover.java:86
org.apache.hadoop.hive.ql.optimizer.Optimizer.optimize() Optimizer.java:209
org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.analyzeInternal(ASTNode,
SemanticAnalyzer$PlannerContext) SemanticAnalyzer.java:10195
org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.analyzeInternal(ASTNode)
SemanticAnalyzer.java:9980
org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(ASTNode, Context)
BaseSemanticAnalyzer.java:240
org.apache.hadoop.hive.ql.Driver.compile(String, boolean) Driver.java:440
org.apache.hadoop.hive.ql.Driver.compile(String) Driver.java:311
org.apache.hadoop.hive.ql.Driver.compileInternal(String) Driver.java:1162
org.apache.hadoop.hive.ql.Driver.runInternal(String, boolean) Driver.java:1215
org.apache.hadoop.hive.ql.Driver.run(String, boolean) Driver.java:1091
org.apache.hadoop.hive.ql.Driver.run(String) Driver.java:1081
org.apache.hadoop.hive.cli.CliDriver.processLocalCmd(String, CommandProcessor,
CliSessionState) CliDriver.java:225
org.apache.hadoop.hive.cli.CliDriver.processCmd(String) CliDriver.java:177
org.apache.hadoop.hive.cli.CliDriver.processLine(String, boolean)
CliDriver.java:388
org.apache.hadoop.hive.cli.CliDriver.executeDriver(CliSessionState, HiveConf,
OptionsProcessor) CliDriver.java:759
org.apache.hadoop.hive.cli.CliDriver.run(String[]) CliDriver.java:704
org.apache.hadoop.hive.cli.CliDriver.main(String[]) CliDriver.java:633
sun.reflect.NativeMethodAccessorImpl.invoke0(Method, Object, Object[])
NativeMethodAccessorImpl.java (native)
sun.reflect.NativeMethodAccessorImpl.invoke(Object, Object[])
NativeMethodAccessorImpl.java:62
sun.reflect.DelegatingMethodAccessorImpl.invoke(Object, Object[])
DelegatingMethodAccessorImpl.java:43
java.lang.reflect.Method.invoke(Object, Object[]) Method.java:497
org.apache.hadoop.util.RunJar.run(String[]) RunJar.java:221
org.apache.hadoop.util.RunJar.main(String[]) RunJar.java:136
{code}
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
