Nitin Pawar created DRILL-7517:
----------------------------------
Summary: Drill 1.16.0 shuts down frequently
Key: DRILL-7517
URL: https://issues.apache.org/jira/browse/DRILL-7517
Project: Apache Drill
Issue Type: Bug
Affects Versions: 1.16.0
Reporter: Nitin Pawar
We see following exception every few hours
Our drillbit cluster queries data from S3.
[Error Id: 1f14f398-fab2-40cd-b14c-389d91c002e8 on 10.0.160.5:31010]
org.apache.drill.common.exceptions.UserException: SYSTEM ERROR:
IllegalStateException: Connection pool shut down
Fragment 1:0
Please, refer to logs for more information.
[Error Id: 1f14f398-fab2-40cd-b14c-389d91c002e8 on 10.0.160.5:31010]
at
org.apache.drill.common.exceptions.UserException$Builder.build(UserException.java:630)
~[drill-common-1.16.0.jar:1.16.0]
at
org.apache.drill.exec.work.fragment.FragmentExecutor.sendFinalState(FragmentExecutor.java:363)
[drill-java-exec-1.16.0.jar:1.16.0]
at
org.apache.drill.exec.work.fragment.FragmentExecutor.cleanup(FragmentExecutor.java:219)
[drill-java-exec-1.16.0.jar:1.16.0]
at
org.apache.drill.exec.work.fragment.FragmentExecutor.run(FragmentExecutor.java:329)
[drill-java-exec-1.16.0.jar:1.16.0]
at
org.apache.drill.common.SelfCleaningRunnable.run(SelfCleaningRunnable.java:38)
[drill-common-1.16.0.jar:1.16.0]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[na:1.8.0_222]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[na:1.8.0_222]
at java.lang.Thread.run(Thread.java:748) [na:1.8.0_222]
Caused by: java.lang.IllegalStateException: Connection pool shut down
at org.apache.http.pool.AbstractConnPool.lease(AbstractConnPool.java:167)
~[httpcore-4.2.4.jar:4.2.4]
at org.apache.http.pool.AbstractConnPool.lease(AbstractConnPool.java:199)
~[httpcore-4.2.4.jar:4.2.4]
at
org.apache.http.impl.conn.PoolingClientConnectionManager.requestConnection(PoolingClientConnectionManager.java:188)
~[httpclient-4.2.5.jar:4.2.5]
at sun.reflect.GeneratedMethodAccessor24.invoke(Unknown Source) ~[na:na]
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
~[na:1.8.0_222]
at java.lang.reflect.Method.invoke(Method.java:498) ~[na:1.8.0_222]
at
com.amazonaws.http.conn.ClientConnectionManagerFactory$Handler.invoke(ClientConnectionManagerFactory.java:72)
~[aws-java-sdk-1.7.4.jar:na]
at com.amazonaws.http.conn.$Proxy76.requestConnection(Unknown Source) ~[na:na]
at
org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:448)
~[httpclient-4.2.5.jar:4.2.5]
at
org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:906)
~[httpclient-4.2.5.jar:4.2.5]
at
org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:805)
~[httpclient-4.2.5.jar:4.2.5]
at
com.amazonaws.http.AmazonHttpClient.executeHelper(AmazonHttpClient.java:384)
~[aws-java-sdk-1.7.4.jar:na]
at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:232)
~[aws-java-sdk-1.7.4.jar:na]
at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:3528)
~[aws-java-sdk-1.7.4.jar:na]
at
com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:976)
~[aws-java-sdk-1.7.4.jar:na]
at
com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:956)
~[aws-java-sdk-1.7.4.jar:na]
at
org.apache.hadoop.fs.s3a.S3AFileSystem.getFileStatus(S3AFileSystem.java:892)
~[hadoop-aws-2.7.4.jar:na]
at org.apache.hadoop.fs.s3a.S3AFileSystem.getFileStatus(S3AFileSystem.java:77)
~[hadoop-aws-2.7.4.jar:na]
at org.apache.hadoop.fs.FileSystem.exists(FileSystem.java:1426)
~[hadoop-common-2.7.4.jar:na]
at
org.apache.drill.exec.store.StorageStrategy.getNonExistentLocations(StorageStrategy.java:200)
~[drill-java-exec-1.16.0.jar:1.16.0]
at
org.apache.drill.exec.store.StorageStrategy.createFileAndApply(StorageStrategy.java:134)
~[drill-java-exec-1.16.0.jar:1.16.0]
at
org.apache.drill.exec.store.text.DrillTextRecordWriter.startNewSchema(DrillTextRecordWriter.java:92)
~[drill-java-exec-1.16.0.jar:1.16.0]
at
org.apache.drill.exec.store.StringOutputRecordWriter.updateSchema(StringOutputRecordWriter.java:44)
~[drill-java-exec-1.16.0.jar:1.16.0]
at
org.apache.drill.exec.physical.impl.WriterRecordBatch.setupNewSchema(WriterRecordBatch.java:160)
~[drill-java-exec-1.16.0.jar:1.16.0]
at
org.apache.drill.exec.physical.impl.WriterRecordBatch.innerNext(WriterRecordBatch.java:108)
~[drill-java-exec-1.16.0.jar:1.16.0]
at
org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:186)
~[drill-java-exec-1.16.0.jar:1.16.0]
at
org.apache.drill.exec.physical.impl.BaseRootExec.next(BaseRootExec.java:104)
~[drill-java-exec-1.16.0.jar:1.16.0]
at
org.apache.drill.exec.physical.impl.SingleSenderCreator$SingleSenderRootExec.innerNext(SingleSenderCreator.java:93)
~[drill-java-exec-1.16.0.jar:1.16.0]
at org.apache.drill.exec.physical.impl.BaseRootExec.next(BaseRootExec.java:94)
~[drill-java-exec-1.16.0.jar:1.16.0]
at
org.apache.drill.exec.work.fragment.FragmentExecutor$1.run(FragmentExecutor.java:296)
~[drill-java-exec-1.16.0.jar:1.16.0]
at
org.apache.drill.exec.work.fragment.FragmentExecutor$1.run(FragmentExecutor.java:283)
~[drill-java-exec-1.16.0.jar:1.16.0]
at java.security.AccessController.doPrivileged(Native Method) ~[na:1.8.0_222]
at javax.security.auth.Subject.doAs(Subject.java:422) ~[na:1.8.0_222]
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1746)
~[hadoop-common-2.7.4.jar:na]
at
org.apache.drill.exec.work.fragment.FragmentExecutor.run(FragmentExecutor.java:283)
[drill-java-exec-1.16.0.jar:1.16.0]
... 4 common frames omitted
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
