Aihua Xu created HIVE-18009:
-------------------------------
Summary: Multiple lateral view query is slow on hive on spark
Key: HIVE-18009
URL: https://issues.apache.org/jira/browse/HIVE-18009
Project: Hive
Issue Type: Improvement
Components: Spark
Affects Versions: 3.0.0
Reporter: Aihua Xu
Assignee: Aihua Xu
When running the query with multiple lateral view, HoS is busy with the
compilation. GenSparkUtils has an efficient implementation of getChildOperator
when we have diamond hierarchy in operator trees (lateral view in this case)
since the node may be visited multiple times.
{noformat}
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:442)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWork.process(GenSparkWork.java:104)
at
org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher.dispatch(DefaultRuleDispatcher.java:90)
at
org.apache.hadoop.hive.ql.lib.DefaultGraphWalker.dispatchAndReturn(DefaultGraphWalker.java:94)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:90)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106)
at
org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.startWalking(GenSparkWorkWalker.java:72)
at
org.apache.hadoop.hive.ql.parse.spark.SparkCompiler.generateTaskTree(SparkCompiler.java:225)
at
org.apache.hadoop.hive.ql.parse.TaskCompiler.compile(TaskCompiler.java:204)
at
org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.analyzeInternal(SemanticAnalyzer.java:10083)
at
org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.analyzeInternal(SemanticAnalyzer.java:9876)
at
org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(BaseSemanticAnalyzer.java:222)
at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:445)
at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:311)
at org.apache.hadoop.hive.ql.Driver.compileInternal(Driver.java:1189)
at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:1284)
at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1115)
at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1103)
at
org.apache.hadoop.hive.cli.CliDriver.processLocalCmd(CliDriver.java:220)
at org.apache.hadoop.hive.cli.CliDriver.processCmd(CliDriver.java:172)
at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:383)
at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:318)
at
org.apache.hadoop.hive.cli.CliDriver.processReader(CliDriver.java:416)
at org.apache.hadoop.hive.cli.CliDriver.processFile(CliDriver.java:432)
at
org.apache.hadoop.hive.cli.CliDriver.executeDriver(CliDriver.java:726)
at org.apache.hadoop.hive.cli.CliDriver.run(CliDriver.java:693)
at org.apache.hadoop.hive.cli.CliDriver.main(CliDriver.java:628)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.hadoop.util.RunJar.run(RunJar.java:221)
at org.apache.hadoop.util.RunJar.main(RunJar.java:136)
{noformat}
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
