Hi, When you state:
"The Queries are dependent on certain table which are created on Runtime(refreshed every hour), .." What is refreshed the whole of DDL for the same table or table is truncated? If these tables are created and dropped within the same session why not use temporary tables (private to that session). A DDL in any database is a heavy operation if you can truncate or overwrite the existing tables it would be prudent. HTH Dr Mich Talebzadeh LinkedIn * https://www.linkedin.com/profile/view?id=AAEAAAAWh2gBxianrbJd6zP6AcPCCdOABUrV8Pw <https://www.linkedin.com/profile/view?id=AAEAAAAWh2gBxianrbJd6zP6AcPCCdOABUrV8Pw>* http://talebzadehmich.wordpress.com On 27 April 2016 at 19:51, Vineet Mishra <clearmido...@gmail.com> wrote: > Hi All, > > I am running Hive Server2 on a big Hadoop cluster space and things looks > good, I have a custom code which takes care of running some queries on top > of Hive and processing the resultant accordingly. > > The Queries are dependent on certain table which are created on > Runtime(refreshed every hour), since yesterday I have included some more > create statements in the code which have the dependency on the next set of > queries, to be more specific the new set of create table statements are > quite costly are spawn around 50+ MR in comparison to existing use case > where only a few 10 MR would be spawned. > > Now since when I have included these heavy DML queries (create, drop) I am > landing into Transport Exception and my HiveServer goes down(Stack Trace > mentioned below). > > java.sql.SQLException: org.apache.thrift.transport.TTransportException > at org.apache.hive.jdbc.HiveStatement.execute(HiveStatement.java:238) > at org.apache.hive.jdbc.HiveStatement.executeUpdate(HiveStatement.java:390) > at > org.apache.hive.jdbc.HivePreparedStatement.executeUpdate(HivePreparedStatement.java:117) > at > com.sd.dwh.sc.tungsten.misc.CommonFunction.executeUpdate(CommonFunction.java:816) > at > com.sd.dwh.sc.tungsten.custom.HiveRunnable.mergeJDBC(HiveRunnable.java:95) > at com.sd.dwh.sc.tungsten.custom.HiveRunnable.run(HiveRunnable.java:62) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) > at java.lang.Thread.run(Thread.java:745) > Caused by: org.apache.thrift.transport.TTransportException > at > org.apache.thrift.transport.TIOStreamTransport.read(TIOStreamTransport.java:132) > at org.apache.thrift.transport.TTransport.readAll(TTransport.java:84) > at > org.apache.thrift.transport.TSaslTransport.readLength(TSaslTransport.java:355) > at > org.apache.thrift.transport.TSaslTransport.readFrame(TSaslTransport.java:432) > at org.apache.thrift.transport.TSaslTransport.read(TSaslTransport.java:414) > at > org.apache.thrift.transport.TSaslClientTransport.read(TSaslClientTransport.java:37) > at org.apache.thrift.transport.TTransport.readAll(TTransport.java:84) > at > org.apache.thrift.protocol.TBinaryProtocol.readAll(TBinaryProtocol.java:378) > at > org.apache.thrift.protocol.TBinaryProtocol.readI32(TBinaryProtocol.java:297) > at > org.apache.thrift.protocol.TBinaryProtocol.readMessageBegin(TBinaryProtocol.java:204) > at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:69) > at > org.apache.hive.service.cli.thrift.TCLIService$Client.recv_ExecuteStatement(TCLIService.java:229) > at > org.apache.hive.service.cli.thrift.TCLIService$Client.ExecuteStatement(TCLIService.java:216) > at org.apache.hive.jdbc.HiveStatement.execute(HiveStatement.java:232) > ... 8 more > java.sql.SQLException: Error while cleaning up the server resources > at org.apache.hive.jdbc.HiveConnection.close(HiveConnection.java:569) > at > com.sd.dwh.sc.tungsten.custom.HiveRunnable.mergeJDBC(HiveRunnable.java:141) > at com.sd.dwh.sc.tungsten.custom.HiveRunnable.run(HiveRunnable.java:62) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) > at java.lang.Thread.run(Thread.java:745) > Caused by: org.apache.thrift.transport.TTransportException > at > org.apache.thrift.transport.TIOStreamTransport.read(TIOStreamTransport.java:132) > at org.apache.thrift.transport.TTransport.readAll(TTransport.java:84) > at > org.apache.thrift.transport.TSaslTransport.readLength(TSaslTransport.java:355) > at > org.apache.thrift.transport.TSaslTransport.readFrame(TSaslTransport.java:432) > at org.apache.thrift.transport.TSaslTransport.read(TSaslTransport.java:414) > at > org.apache.thrift.transport.TSaslClientTransport.read(TSaslClientTransport.java:37) > at org.apache.thrift.transport.TTransport.readAll(TTransport.java:84) > at > org.apache.thrift.protocol.TBinaryProtocol.readAll(TBinaryProtocol.java:378) > at > org.apache.thrift.protocol.TBinaryProtocol.readI32(TBinaryProtocol.java:297) > at > org.apache.thrift.protocol.TBinaryProtocol.readMessageBegin(TBinaryProtocol.java:204) > at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:69) > at > org.apache.hive.service.cli.thrift.TCLIService$Client.recv_CloseSession(TCLIService.java:183) > at > org.apache.hive.service.cli.thrift.TCLIService$Client.CloseSession(TCLIService.java:170) > at org.apache.hive.jdbc.HiveConnection.close(HiveConnection.java:567) > ... 5 more > java.sql.SQLException: org.apache.thrift.transport.TTransportException > at org.apache.hive.jdbc.HiveStatement.execute(HiveStatement.java:238) > at org.apache.hive.jdbc.HiveStatement.executeUpdate(HiveStatement.java:390) > at > org.apache.hive.jdbc.HivePreparedStatement.executeUpdate(HivePreparedStatement.java:117) > at > com.sd.dwh.sc.tungsten.misc.CommonFunction.executeUpdate(CommonFunction.java:816) > at > com.sd.dwh.sc.tungsten.custom.HiveRunnable.mergeJDBC(HiveRunnable.java:95) > at com.sd.dwh.sc.tungsten.custom.HiveRunnable.run(HiveRunnable.java:62) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) > at java.lang.Thread.run(Thread.java:745) > Caused by: org.apache.thrift.transport.TTransportException > at > org.apache.thrift.transport.TIOStreamTransport.read(TIOStreamTransport.java:132) > at org.apache.thrift.transport.TTransport.readAll(TTransport.java:84) > at > org.apache.thrift.transport.TSaslTransport.readLength(TSaslTransport.java:355) > at > org.apache.thrift.transport.TSaslTransport.readFrame(TSaslTransport.java:432) > at org.apache.thrift.transport.TSaslTransport.read(TSaslTransport.java:414) > at > org.apache.thrift.transport.TSaslClientTransport.read(TSaslClientTransport.java:37) > at org.apache.thrift.transport.TTransport.readAll(TTransport.java:84) > at > org.apache.thrift.protocol.TBinaryProtocol.readAll(TBinaryProtocol.java:378) > at > org.apache.thrift.protocol.TBinaryProtocol.readI32(TBinaryProtocol.java:297) > at > org.apache.thrift.protocol.TBinaryProtocol.readMessageBegin(TBinaryProtocol.java:204) > at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:69) > at > org.apache.hive.service.cli.thrift.TCLIService$Client.recv_ExecuteStatement(TCLIService.java:229) > at > org.apache.hive.service.cli.thrift.TCLIService$Client.ExecuteStatement(TCLIService.java:216) > at org.apache.hive.jdbc.HiveStatement.execute(HiveStatement.java:232) > ... 8 more > java.sql.SQLException: Error while cleaning up the server resources > at org.apache.hive.jdbc.HiveConnection.close(HiveConnection.java:569) > at > com.sd.dwh.sc.tungsten.custom.HiveRunnable.mergeJDBC(HiveRunnable.java:141) > at com.sd.dwh.sc.tungsten.custom.HiveRunnable.run(HiveRunnable.java:62) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) > at java.lang.Thread.run(Thread.java:745) > Caused by: org.apache.thrift.transport.TTransportException: > java.net.SocketException: Broken pipe > at > org.apache.thrift.transport.TIOStreamTransport.flush(TIOStreamTransport.java:161) > at > org.apache.thrift.transport.TSaslTransport.flush(TSaslTransport.java:480) > at > org.apache.thrift.transport.TSaslClientTransport.flush(TSaslClientTransport.java:37) > at org.apache.thrift.TServiceClient.sendBase(TServiceClient.java:65) > at > org.apache.hive.service.cli.thrift.TCLIService$Client.send_CloseSession(TCLIService.java:177) > at > org.apache.hive.service.cli.thrift.TCLIService$Client.CloseSession(TCLIService.java:169) > at org.apache.hive.jdbc.HiveConnection.close(HiveConnection.java:567) > ... 5 more > Caused by: java.net.SocketException: Broken pipe > at java.net.SocketOutputStream.socketWrite0(Native Method) > at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:113) > at java.net.SocketOutputStream.write(SocketOutputStream.java:159) > at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) > at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) > at > org.apache.thrift.transport.TIOStreamTransport.flush(TIOStreamTransport.java:159) > ... 11 more > java.sql.SQLException: org.apache.thrift.transport.TTransportException > at org.apache.hive.jdbc.HiveStatement.execute(HiveStatement.java:238) > at org.apache.hive.jdbc.HiveStatement.executeUpdate(HiveStatement.java:390) > at > org.apache.hive.jdbc.HivePreparedStatement.executeUpdate(HivePreparedStatement.java:117) > at > com.sd.dwh.sc.tungsten.misc.CommonFunction.executeUpdate(CommonFunction.java:816) > at > com.sd.dwh.sc.tungsten.custom.HiveRunnable.mergeJDBC(HiveRunnable.java:95) > at com.sd.dwh.sc.tungsten.custom.HiveRunnable.run(HiveRunnable.java:62) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) > at java.lang.Thread.run(Thread.java:745) > Caused by: org.apache.thrift.transport.TTransportException > at > org.apache.thrift.transport.TIOStreamTransport.read(TIOStreamTransport.java:132) > at org.apache.thrift.transport.TTransport.readAll(TTransport.java:84) > at > org.apache.thrift.transport.TSaslTransport.readLength(TSaslTransport.java:355) > at > org.apache.thrift.transport.TSaslTransport.readFrame(TSaslTransport.java:432) > at org.apache.thrift.transport.TSaslTransport.read(TSaslTransport.java:414) > at > org.apache.thrift.transport.TSaslClientTransport.read(TSaslClientTransport.java:37) > at org.apache.thrift.transport.TTransport.readAll(TTransport.java:84) > at > org.apache.thrift.protocol.TBinaryProtocol.readAll(TBinaryProtocol.java:378) > at > org.apache.thrift.protocol.TBinaryProtocol.readI32(TBinaryProtocol.java:297) > at > org.apache.thrift.protocol.TBinaryProtocol.readMessageBegin(TBinaryProtocol.java:204) > at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:69) > at > org.apache.hive.service.cli.thrift.TCLIService$Client.recv_ExecuteStatement(TCLIService.java:229) > at > org.apache.hive.service.cli.thrift.TCLIService$Client.ExecuteStatement(TCLIService.java:216) > at org.apache.hive.jdbc.HiveStatement.execute(HiveStatement.java:232) > ... 8 more > java.sql.SQLException: Error while cleaning up the server resources > at org.apache.hive.jdbc.HiveConnection.close(HiveConnection.java:569) > at > com.sd.dwh.sc.tungsten.custom.HiveRunnable.mergeJDBC(HiveRunnable.java:141) > at com.sd.dwh.sc.tungsten.custom.HiveRunnable.run(HiveRunnable.java:62) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) > at java.lang.Thread.run(Thread.java:745) > Caused by: org.apache.thrift.transport.TTransportException: > java.net.SocketException: Broken pipe > at > org.apache.thrift.transport.TIOStreamTransport.flush(TIOStreamTransport.java:161) > at > org.apache.thrift.transport.TSaslTransport.flush(TSaslTransport.java:480) > at > org.apache.thrift.transport.TSaslClientTransport.flush(TSaslClientTransport.java:37) > at org.apache.thrift.TServiceClient.sendBase(TServiceClient.java:65) > at > org.apache.hive.service.cli.thrift.TCLIService$Client.send_CloseSession(TCLIService.java:177) > at > org.apache.hive.service.cli.thrift.TCLIService$Client.CloseSession(TCLIService.java:169) > at org.apache.hive.jdbc.HiveConnection.close(HiveConnection.java:567) > ... 5 more > Caused by: java.net.SocketException: Broken pipe > at java.net.SocketOutputStream.socketWrite0(Native Method) > at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:113) > at java.net.SocketOutputStream.write(SocketOutputStream.java:159) > at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) > at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) > at > org.apache.thrift.transport.TIOStreamTransport.flush(TIOStreamTransport.java:159) > ... 11 more > java.sql.SQLException: org.apache.thrift.transport.TTransportException > at org.apache.hive.jdbc.HiveStatement.execute(HiveStatement.java:238) > at org.apache.hive.jdbc.HiveStatement.executeQuery(HiveStatement.java:376) > at > org.apache.hive.jdbc.HivePreparedStatement.executeQuery(HivePreparedStatement.java:107) > at > com.sd.dwh.sc.tungsten.misc.CommonFunction.runQuery(CommonFunction.java:771) > at > com.sd.dwh.sc.tungsten.custom.HiveRunnable.mergeJDBC(HiveRunnable.java:113) > at com.sd.dwh.sc.tungsten.custom.HiveRunnable.run(HiveRunnable.java:62) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) > at java.lang.Thread.run(Thread.java:745) > Caused by: org.apache.thrift.transport.TTransportException > at > org.apache.thrift.transport.TIOStreamTransport.read(TIOStreamTransport.java:132) > at org.apache.thrift.transport.TTransport.readAll(TTransport.java:84) > at > org.apache.thrift.transport.TSaslTransport.readLength(TSaslTransport.java:355) > at > org.apache.thrift.transport.TSaslTransport.readFrame(TSaslTransport.java:432) > at org.apache.thrift.transport.TSaslTransport.read(TSaslTransport.java:414) > at > org.apache.thrift.transport.TSaslClientTransport.read(TSaslClientTransport.java:37) > at org.apache.thrift.transport.TTransport.readAll(TTransport.java:84) > at > org.apache.thrift.protocol.TBinaryProtocol.readAll(TBinaryProtocol.java:378) > at > org.apache.thrift.protocol.TBinaryProtocol.readI32(TBinaryProtocol.java:297) > at > org.apache.thrift.protocol.TBinaryProtocol.readMessageBegin(TBinaryProtocol.java:204) > at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:69) > at > org.apache.hive.service.cli.thrift.TCLIService$Client.recv_ExecuteStatement(TCLIService.java:229) > at > org.apache.hive.service.cli.thrift.TCLIService$Client.ExecuteStatement(TCLIService.java:216) > at org.apache.hive.jdbc.HiveStatement.execute(HiveStatement.java:232) > ... 8 more > java.sql.SQLException: Error while cleaning up the server resources > at org.apache.hive.jdbc.HiveConnection.close(HiveConnection.java:569) > at > com.sd.dwh.sc.tungsten.custom.HiveRunnable.mergeJDBC(HiveRunnable.java:141) > at com.sd.dwh.sc.tungsten.custom.HiveRunnable.run(HiveRunnable.java:62) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) > at java.lang.Thread.run(Thread.java:745) > Caused by: org.apache.thrift.transport.TTransportException: > java.net.SocketException: Broken pipe > at > org.apache.thrift.transport.TIOStreamTransport.flush(TIOStreamTransport.java:161) > at > org.apache.thrift.transport.TSaslTransport.flush(TSaslTransport.java:480) > at > org.apache.thrift.transport.TSaslClientTransport.flush(TSaslClientTransport.java:37) > at org.apache.thrift.TServiceClient.sendBase(TServiceClient.java:65) > at > org.apache.hive.service.cli.thrift.TCLIService$Client.send_CloseSession(TCLIService.java:177) > at > org.apache.hive.service.cli.thrift.TCLIService$Client.CloseSession(TCLIService.java:169) > at org.apache.hive.jdbc.HiveConnection.close(HiveConnection.java:567) > ... 5 more > Caused by: java.net.SocketException: Broken pipe > at java.net.SocketOutputStream.socketWrite0(Native Method) > at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:113) > at java.net.SocketOutputStream.write(SocketOutputStream.java:159) > at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) > at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) > at > org.apache.thrift.transport.TIOStreamTransport.flush(TIOStreamTransport.java:159) > ... 11 more > > > Any suggestion would be appreciated! > > Thanks! >
