Hive Version: 2.1.0
I have a very large, multi-line input query (8,668,519 chars) and I have
gone up to 16g heap and still get the same OOM.
Error: Error running query: java.lang.OutOfMemoryError: Java heap space
(state=,code=0)
org.apache.hive.service.cli.HiveSQLException: Error running query:
java.lang.OutOfMemoryError: Java heap space
at org.apache.hive.jdbc.Utils.verifySuccess(Utils.java:264)
at org.apache.hive.jdbc.Utils.verifySuccessWithInfo(Utils.java:250)
at org.apache.hive.jdbc.HiveStatement.runAsyncOnServer(
HiveStatement.java:309)
at org.apache.hive.jdbc.HiveStatement.execute(
HiveStatement.java:250)
at org.apache.hive.beeline.Commands.executeInternal(
Commands.java:977)
at org.apache.hive.beeline.Commands.execute(Commands.java:1148)
at org.apache.hive.beeline.Commands.sql(Commands.java:1063)
at org.apache.hive.beeline.BeeLine.dispatch(BeeLine.java:1134)
at org.apache.hive.beeline.BeeLine.execute(BeeLine.java:965)
at org.apache.hive.beeline.BeeLine.executeFile(BeeLine.java:940)
at org.apache.hive.beeline.BeeLine.begin(BeeLine.java:867)
at org.apache.hive.beeline.BeeLine.mainWithInputRedirection(
BeeLine.java:499)
at org.apache.hive.beeline.BeeLine.main(BeeLine.java:482)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(
NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(
DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.hadoop.util.RunJar.run(RunJar.java:221)
at org.apache.hadoop.util.RunJar.main(RunJar.java:136)
Caused by: org.apache.hive.service.cli.HiveSQLException: Error running
query: java.lang.OutOfMemoryError: Java heap space
at org.apache.hive.service.cli.operation.SQLOperation.
prepare(SQLOperation.java:218)
at org.apache.hive.service.cli.operation.SQLOperation.
runInternal(SQLOperation.java:269)
at org.apache.hive.service.cli.operation.Operation.run(
Operation.java:324)
at org.apache.hive.service.cli.session.HiveSessionImpl.
executeStatementInternal(HiveSessionImpl.java:460)
at org.apache.hive.service.cli.session.HiveSessionImpl.
executeStatementAsync(HiveSessionImpl.java:447)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(
NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(
DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.hive.service.cli.session.HiveSessionProxy.
invoke(HiveSessionProxy.java:78)
at org.apache.hive.service.cli.session.HiveSessionProxy.
access$000(HiveSessionProxy.java:36)
at org.apache.hive.service.cli.session.HiveSessionProxy$1.
run(HiveSessionProxy.java:63)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(
UserGroupInformation.java:1657)
at org.apache.hive.service.cli.session.HiveSessionProxy.
invoke(HiveSessionProxy.java:59)
at com.sun.proxy.$Proxy33.executeStatementAsync(Unknown Source)
at org.apache.hive.service.cli.CLIService.executeStatementAsync(
CLIService.java:294)
at org.apache.hive.service.cli.thrift.ThriftCLIService.
ExecuteStatement(ThriftCLIService.java:497)
at org.apache.hive.service.rpc.thrift.TCLIService$Processor$
ExecuteStatement.getResult(TCLIService.java:1437)
at org.apache.hive.service.rpc.thrift.TCLIService$Processor$
ExecuteStatement.getResult(TCLIService.java:1422)
at org.apache.thrift.ProcessFunction.process(
ProcessFunction.java:39)
at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39)
at org.apache.hive.service.auth.TSetIpAddressProcessor.process(
TSetIpAddressProcessor.java:56)
at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(
TThreadPoolServer.java:286)
at java.util.concurrent.ThreadPoolExecutor.runWorker(
ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(
ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.OutOfMemoryError: Java heap space
at java.util.Arrays.copyOf(Arrays.java:3332)
at java.lang.AbstractStringBuilder.ensureCapacityInternal(
AbstractStringBuilder.java:124)
at java.lang.AbstractStringBuilder.append(
AbstractStringBuilder.java:448)
at java.lang.StringBuffer.append(StringBuffer.java:270)
at java.io.StringWriter.write(StringWriter.java:112)
at java.io.PrintWriter.write(PrintWriter.java:456)
at java.io.PrintWriter.write(PrintWriter.java:473)
at org.apache.calcite.rel.AbstractRelNode$1.explain_(
AbstractRelNode.java:409)
at org.apache.calcite.rel.externalize.RelWriterImpl.
done(RelWriterImpl.java:157)
at org.apache.calcite.rel.AbstractRelNode.explain(
AbstractRelNode.java:308)
at org.apache.calcite.rel.AbstractRelNode.computeDigest(
AbstractRelNode.java:416)
at org.apache.calcite.rel.AbstractRelNode.recomputeDigest(
AbstractRelNode.java:352)
at org.apache.calcite.plan.hep.HepPlanner.buildFinalPlan(
HepPlanner.java:881)
at org.apache.calcite.plan.hep.HepPlanner.findBestExp(
HepPlanner.java:199)
at org.apache.hadoop.hive.ql.parse.CalcitePlanner$
CalcitePlannerAction.hepPlan(CalcitePlanner.java:1309)
at org.apache.hadoop.hive.ql.parse.CalcitePlanner$
CalcitePlannerAction.hepPlan(CalcitePlanner.java:1263)
at org.apache.hadoop.hive.ql.parse.CalcitePlanner$
CalcitePlannerAction.applyPreJoinOrderingTransforms
(CalcitePlanner.java:1130)
at org.apache.hadoop.hive.ql.parse.CalcitePlanner$
CalcitePlannerAction.apply(CalcitePlanner.java:962)
at org.apache.hadoop.hive.ql.parse.CalcitePlanner$
CalcitePlannerAction.apply(CalcitePlanner.java:893)
at org.apache.calcite.tools.Frameworks$1.apply(Frameworks.java:113)
at org.apache.calcite.prepare.CalcitePrepareImpl.perform(
CalcitePrepareImpl.java:969)
at org.apache.calcite.tools.Frameworks.withPrepare(
Frameworks.java:149)
at org.apache.calcite.tools.Frameworks.withPlanner(
Frameworks.java:106)
at org.apache.hadoop.hive.ql.parse.CalcitePlanner.getOptimizedAST(
CalcitePlanner.java:712)
at org.apache.hadoop.hive.ql.parse.CalcitePlanner.
genOPTree(CalcitePlanner.java:280)
at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.analyzeInternal(
SemanticAnalyzer.java:10755)
at org.apache.hadoop.hive.ql.parse.CalcitePlanner.analyzeInternal(
CalcitePlanner.java:239)
at org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.
analyze(BaseSemanticAnalyzer.java:250)
at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:437)
at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:329)
at org.apache.hadoop.hive.ql.Driver.compileInternal(Driver.
java:1158)
at org.apache.hadoop.hive.ql.Driver.compileAndRespond(
Driver.java:1145)
