[ https://issues.apache.org/jira/browse/DRILL-4708?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16180056#comment-16180056 ]
Lenin edited comment on DRILL-4708 at 9/26/17 1:30 AM: ------------------------------------------------------- With Debug log level in the client side : {code} 2017-09-25 16:49:01,764 [Client-1] ERROR o.a.d.exec.rpc.RpcExceptionHandler - Exception in RPC communication. Connection: /192.168.0.106:58493 <--> /192.168.0.106:31010 (user client). Closing connection. io.netty.handler.codec.DecoderException: org.apache.drill.exec.exception.OutOfMemoryException: Failure allocating buffer. at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:233) ~[netty-codec-4.0.27.Final.jar:4.0.27.Final] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:339) [netty-transport-4.0.27.Final.jar:4.0.27.Final] at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:324) [netty-transport-4.0.27.Final.jar:4.0.27.Final] at io.netty.channel.ChannelInboundHandlerAdapter.channelRead(ChannelInboundHandlerAdapter.java:86) [netty-transport-4.0.27.Final.jar:4.0.27.Final] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:339) [netty-transport-4.0.27.Final.jar:4.0.27.Final] at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:324) [netty-transport-4.0.27.Final.jar:4.0.27.Final] at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:847) [netty-transport-4.0.27.Final.jar:4.0.27.Final] at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:131) [netty-transport-4.0.27.Final.jar:4.0.27.Final] at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:511) [netty-transport-4.0.27.Final.jar:4.0.27.Final] at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:468) [netty-transport-4.0.27.Final.jar:4.0.27.Final] at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:382) [netty-transport-4.0.27.Final.jar:4.0.27.Final] at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:354) [netty-transport-4.0.27.Final.jar:4.0.27.Final] at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:111) [netty-common-4.0.27.Final.jar:4.0.27.Final] at java.lang.Thread.run(Thread.java:745) [na:1.7.0_67] Caused by: org.apache.drill.exec.exception.OutOfMemoryException: Failure allocating buffer. at io.netty.buffer.PooledByteBufAllocatorL.allocate(PooledByteBufAllocatorL.java:64) ~[drill-memory-base-1.11.0.jar:4.0.27.Final] at org.apache.drill.exec.memory.AllocationManager.<init>(AllocationManager.java:80) ~[drill-memory-base-1.11.0.jar:1.11.0] at org.apache.drill.exec.memory.BaseAllocator.bufferWithoutReservation(BaseAllocator.java:260) ~[drill-memory-base-1.11.0.jar:1.11.0] at org.apache.drill.exec.memory.BaseAllocator.buffer(BaseAllocator.java:243) ~[drill-memory-base-1.11.0.jar:1.11.0] at org.apache.drill.exec.memory.BaseAllocator.buffer(BaseAllocator.java:213) ~[drill-memory-base-1.11.0.jar:1.11.0] at io.netty.buffer.ExpandableByteBuf.capacity(ExpandableByteBuf.java:43) ~[drill-memory-base-1.11.0.jar:4.0.27.Final] at io.netty.buffer.AbstractByteBuf.ensureWritable(AbstractByteBuf.java:251) ~[netty-buffer-4.0.27.Final.jar:4.0.27.Final] at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:849) ~[netty-buffer-4.0.27.Final.jar:4.0.27.Final] at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:841) ~[netty-buffer-4.0.27.Final.jar:4.0.27.Final] at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:831) ~[netty-buffer-4.0.27.Final.jar:4.0.27.Final] at io.netty.handler.codec.ByteToMessageDecoder$1.cumulate(ByteToMessageDecoder.java:92) ~[netty-codec-4.0.27.Final.jar:4.0.27.Final] at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:227) ~[netty-codec-4.0.27.Final.jar:4.0.27.Final] ... 13 common frames omitted Caused by: java.lang.OutOfMemoryError: Direct buffer memory at java.nio.Bits.reserveMemory(Bits.java:658) ~[na:1.7.0_67] at java.nio.DirectByteBuffer.<init>(DirectByteBuffer.java:123) ~[na:1.7.0_67] at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:306) ~[na:1.7.0_67] at io.netty.buffer.UnpooledUnsafeDirectByteBuf.allocateDirect(UnpooledUnsafeDirectByteBuf.java:108) ~[netty-buffer-4.0.27.Final.jar:4.0.27.Final] at io.netty.buffer.UnpooledUnsafeDirectByteBuf.<init>(UnpooledUnsafeDirectByteBuf.java:69) ~[netty-buffer-4.0.27.Final.jar:4.0.27.Final] at io.netty.buffer.UnpooledByteBufAllocator.newDirectBuffer(UnpooledByteBufAllocator.java:50) ~[netty-buffer-4.0.27.Final.jar:4.0.27.Final] at io.netty.buffer.AbstractByteBufAllocator.directBuffer(AbstractByteBufAllocator.java:155) ~[netty-buffer-4.0.27.Final.jar:4.0.27.Final] at io.netty.buffer.PooledByteBufAllocatorL$InnerAllocator.newDirectBufferL(PooledByteBufAllocatorL.java:155) ~[drill-memory-base-1.11.0.jar:4.0.27.Final] at io.netty.buffer.PooledByteBufAllocatorL$InnerAllocator.directBuffer(PooledByteBufAllocatorL.java:195) ~[drill-memory-base-1.11.0.jar:4.0.27.Final] at io.netty.buffer.PooledByteBufAllocatorL.allocate(PooledByteBufAllocatorL.java:62) ~[drill-memory-base-1.11.0.jar:4.0.27.Final] ... 24 common frames omitted 2017-09-25 16:49:01,792 [Client-1] INFO o.a.d.e.rpc.user.QueryResultHandler - User Error Occurred: Connection /192.168.0.106:58493 <--> /192.168.0.106:31010 (user client) closed unexpectedly. Drillbit down? org.apache.drill.common.exceptions.UserException: CONNECTION ERROR: Connection /192.168.0.106:58493 <--> /192.168.0.106:31010 (user client) closed unexpectedly. Drillbit down? [Error Id: 07a79a3d-d185-44a3-b3ab-fee2f85376f7 ] at org.apache.drill.common.exceptions.UserException$Builder.build(UserException.java:550) ~[drill-common-1.11.0.jar:1.11.0] at org.apache.drill.exec.rpc.user.QueryResultHandler$ChannelClosedHandler$1.operationComplete(QueryResultHandler.java:373) [drill-java-exec-1.11.0.jar:1.11.0] at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:680) [netty-common-4.0.27.Final.jar:4.0.27.Final] at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:603) [netty-common-4.0.27.Final.jar:4.0.27.Final] {code} Not sure why client is expected to have Direct memory configured ? was (Author: gitfy): With Debug log level in the client side : {code} 2017-09-25 16:49:01,764 [Client-1] ERROR o.a.d.exec.rpc.RpcExceptionHandler - Exception in RPC communication. Connection: /192.168.0.106:58493 <--> /192.168.0.106:31010 (user client). Closing connection. io.netty.handler.codec.DecoderException: org.apache.drill.exec.exception.OutOfMemoryException: Failure allocating buffer. at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:233) ~[netty-codec-4.0.27.Final.jar:4.0.27.Final] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:339) [netty-transport-4.0.27.Final.jar:4.0.27.Final] at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:324) [netty-transport-4.0.27.Final.jar:4.0.27.Final] at io.netty.channel.ChannelInboundHandlerAdapter.channelRead(ChannelInboundHandlerAdapter.java:86) [netty-transport-4.0.27.Final.jar:4.0.27.Final] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:339) [netty-transport-4.0.27.Final.jar:4.0.27.Final] at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:324) [netty-transport-4.0.27.Final.jar:4.0.27.Final] at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:847) [netty-transport-4.0.27.Final.jar:4.0.27.Final] at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:131) [netty-transport-4.0.27.Final.jar:4.0.27.Final] at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:511) [netty-transport-4.0.27.Final.jar:4.0.27.Final] at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:468) [netty-transport-4.0.27.Final.jar:4.0.27.Final] at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:382) [netty-transport-4.0.27.Final.jar:4.0.27.Final] at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:354) [netty-transport-4.0.27.Final.jar:4.0.27.Final] at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:111) [netty-common-4.0.27.Final.jar:4.0.27.Final] at java.lang.Thread.run(Thread.java:745) [na:1.7.0_67] Caused by: org.apache.drill.exec.exception.OutOfMemoryException: Failure allocating buffer. at io.netty.buffer.PooledByteBufAllocatorL.allocate(PooledByteBufAllocatorL.java:64) ~[drill-memory-base-1.11.0.jar:4.0.27.Final] at org.apache.drill.exec.memory.AllocationManager.<init>(AllocationManager.java:80) ~[drill-memory-base-1.11.0.jar:1.11.0] at org.apache.drill.exec.memory.BaseAllocator.bufferWithoutReservation(BaseAllocator.java:260) ~[drill-memory-base-1.11.0.jar:1.11.0] at org.apache.drill.exec.memory.BaseAllocator.buffer(BaseAllocator.java:243) ~[drill-memory-base-1.11.0.jar:1.11.0] at org.apache.drill.exec.memory.BaseAllocator.buffer(BaseAllocator.java:213) ~[drill-memory-base-1.11.0.jar:1.11.0] at io.netty.buffer.ExpandableByteBuf.capacity(ExpandableByteBuf.java:43) ~[drill-memory-base-1.11.0.jar:4.0.27.Final] at io.netty.buffer.AbstractByteBuf.ensureWritable(AbstractByteBuf.java:251) ~[netty-buffer-4.0.27.Final.jar:4.0.27.Final] at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:849) ~[netty-buffer-4.0.27.Final.jar:4.0.27.Final] at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:841) ~[netty-buffer-4.0.27.Final.jar:4.0.27.Final] at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:831) ~[netty-buffer-4.0.27.Final.jar:4.0.27.Final] at io.netty.handler.codec.ByteToMessageDecoder$1.cumulate(ByteToMessageDecoder.java:92) ~[netty-codec-4.0.27.Final.jar:4.0.27.Final] at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:227) ~[netty-codec-4.0.27.Final.jar:4.0.27.Final] ... 13 common frames omitted Caused by: java.lang.OutOfMemoryError: Direct buffer memory at java.nio.Bits.reserveMemory(Bits.java:658) ~[na:1.7.0_67] at java.nio.DirectByteBuffer.<init>(DirectByteBuffer.java:123) ~[na:1.7.0_67] at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:306) ~[na:1.7.0_67] at io.netty.buffer.UnpooledUnsafeDirectByteBuf.allocateDirect(UnpooledUnsafeDirectByteBuf.java:108) ~[netty-buffer-4.0.27.Final.jar:4.0.27.Final] at io.netty.buffer.UnpooledUnsafeDirectByteBuf.<init>(UnpooledUnsafeDirectByteBuf.java:69) ~[netty-buffer-4.0.27.Final.jar:4.0.27.Final] at io.netty.buffer.UnpooledByteBufAllocator.newDirectBuffer(UnpooledByteBufAllocator.java:50) ~[netty-buffer-4.0.27.Final.jar:4.0.27.Final] at io.netty.buffer.AbstractByteBufAllocator.directBuffer(AbstractByteBufAllocator.java:155) ~[netty-buffer-4.0.27.Final.jar:4.0.27.Final] at io.netty.buffer.PooledByteBufAllocatorL$InnerAllocator.newDirectBufferL(PooledByteBufAllocatorL.java:155) ~[drill-memory-base-1.11.0.jar:4.0.27.Final] at io.netty.buffer.PooledByteBufAllocatorL$InnerAllocator.directBuffer(PooledByteBufAllocatorL.java:195) ~[drill-memory-base-1.11.0.jar:4.0.27.Final] at io.netty.buffer.PooledByteBufAllocatorL.allocate(PooledByteBufAllocatorL.java:62) ~[drill-memory-base-1.11.0.jar:4.0.27.Final] ... 24 common frames omitted {code} Not sure why client is expected to have Direct memory configured ? > connection closed unexpectedly > ------------------------------ > > Key: DRILL-4708 > URL: https://issues.apache.org/jira/browse/DRILL-4708 > Project: Apache Drill > Issue Type: Bug > Components: Execution - RPC > Affects Versions: 1.7.0 > Reporter: Chun Chang > Priority: Critical > > Running DRILL functional automation, we often see query failed randomly due > to the following unexpected connection close error. > {noformat} > Execution Failures: > /root/drillAutomation/framework/framework/resources/Functional/ctas/ctas_flatten/100000rows/filter5.q > Query: > select * from dfs.ctas_flatten.`filter5_100000rows_ctas` > Failed with exception > java.sql.SQLException: CONNECTION ERROR: Connection /10.10.100.171:36185 <--> > drillats4.qa.lab/10.10.100.174:31010 (user client) closed unexpectedly. > Drillbit down? > [Error Id: 3d5dad8e-80d0-4c7f-9012-013bf01ce2b7 ] > at > org.apache.drill.jdbc.impl.DrillCursor.nextRowInternally(DrillCursor.java:247) > at org.apache.drill.jdbc.impl.DrillCursor.next(DrillCursor.java:321) > at > oadd.net.hydromatic.avatica.AvaticaResultSet.next(AvaticaResultSet.java:187) > at > org.apache.drill.jdbc.impl.DrillResultSetImpl.next(DrillResultSetImpl.java:172) > at > org.apache.drill.test.framework.DrillTestJdbc.executeQuery(DrillTestJdbc.java:210) > at > org.apache.drill.test.framework.DrillTestJdbc.run(DrillTestJdbc.java:99) > at > java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) > at java.util.concurrent.FutureTask.run(FutureTask.java:262) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) > at java.lang.Thread.run(Thread.java:744) > Caused by: oadd.org.apache.drill.common.exceptions.UserException: CONNECTION > ERROR: Connection /10.10.100.171:36185 <--> > drillats4.qa.lab/10.10.100.174:31010 (user client) closed unexpectedly. > Drillbit down? > [Error Id: 3d5dad8e-80d0-4c7f-9012-013bf01ce2b7 ] > at > oadd.org.apache.drill.common.exceptions.UserException$Builder.build(UserException.java:543) > at > oadd.org.apache.drill.exec.rpc.user.QueryResultHandler$ChannelClosedHandler$1.operationComplete(QueryResultHandler.java:373) > at > oadd.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:680) > at > oadd.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:603) > at > oadd.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:563) > at > oadd.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:406) > at > oadd.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:82) > at > oadd.io.netty.channel.AbstractChannel$CloseFuture.setClosed(AbstractChannel.java:943) > at > oadd.io.netty.channel.AbstractChannel$AbstractUnsafe.doClose0(AbstractChannel.java:592) > at > oadd.io.netty.channel.AbstractChannel$AbstractUnsafe.close(AbstractChannel.java:584) > at > oadd.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.closeOnRead(AbstractNioByteChannel.java:71) > at > oadd.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.handleReadException(AbstractNioByteChannel.java:89) > at > oadd.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:162) > at > oadd.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:511) > at > oadd.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:468) > at > oadd.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:382) > at oadd.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:354) > at > oadd.io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:111) > ... 1 more > {noformat} -- This message was sent by Atlassian JIRA (v6.4.14#64029)