This is an application that is constantly polling Amazon S3 using the
jets3t Java wrapper.
Attached is a stack trace of one of the recent failures.
-George
[WARN ] [12 Jul 2009 00:53:49.358] [RestUtils] [Retried connection 6
times, which exceeds the maximum retry count of 5]
[ERROR] [12 Jul 2009 00:53:49.358] [AmazonS3]
[org.jets3t.service.S3ServiceException: S3 GET connection failed for
'/']
org.jets3t.service.S3ServiceException: S3 GET connection failed for '/'
at
org
.jets3t
.service
.impl.rest.httpclient.RestS3Service.performRequest(RestS3Service.java:
516)
at
org
.jets3t
.service
.impl.rest.httpclient.RestS3Service.performRestGet(RestS3Service.java:
752)
at
org
.jets3t
.service
.impl
.rest.httpclient.RestS3Service.listAllBucketsImpl(RestS3Service.java:
1041)
at org.jets3t.service.S3Service.listAllBuckets(S3Service.java:1348)
at com.enstratus.provider.aws.AmazonS3.listFiles(AmazonS3.java:406)
at
com
.enstratus
.provisioning.storage.StorageFactory.checkState(StorageFactory.java:55)
at com.enstratus.monitor.CloudMonitor$1.call(CloudMonitor.java:277)
at com.enstratus.monitor.CloudMonitor$1.call(CloudMonitor.java:276)
at com.enstratus.util.Retry.retry(Retry.java:32)
at com.enstratus.util.Retry.retry(Retry.java:19)
at com.enstratus.monitor.CloudMonitor.checkStatus(CloudMonitor.java:
282)
at com.enstratus.monitor.CloudMonitor.access$000(CloudMonitor.java:41)
at com.enstratus.monitor.CloudMonitor$2$1.run(CloudMonitor.java:468)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:
441)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at java.util.concurrent.ThreadPoolExecutor
$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor
$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:619)
Caused by: java.net.SocketException: Too many open files
at java.net.Socket.createImpl(Socket.java:388)
at java.net.Socket.getImpl(Socket.java:451)
at java.net.Socket.bind(Socket.java:565)
at
com
.sun
.net.ssl.internal.ssl.BaseSSLSocketImpl.bind(BaseSSLSocketImpl.java:95)
at com.sun.net.ssl.internal.ssl.SSLSocketImpl.bind(SSLSocketImpl.java:
45)
at sun.reflect.GeneratedMethodAccessor31.invoke(Unknown Source)
at
sun
.reflect
.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:
25)
at java.lang.reflect.Method.invoke(Method.java:597)
at
org
.apache
.commons
.httpclient
.protocol
.ReflectionSocketFactory.createSocket(ReflectionSocketFactory.java:139)
at
org
.apache
.commons
.httpclient
.protocol
.SSLProtocolSocketFactory.createSocket(SSLProtocolSocketFactory.java:
130)
at
org.apache.commons.httpclient.HttpConnection.open(HttpConnection.java:
707)
at org.apache.commons.httpclient.MultiThreadedHttpConnectionManager
$HttpConnectionAdapter.open(MultiThreadedHttpConnectionManager.java:
1361)
at
org
.apache
.commons
.httpclient
.HttpMethodDirector.executeWithRetry(HttpMethodDirector.java:387)
at
org
.apache
.commons
.httpclient.HttpMethodDirector.executeMethod(HttpMethodDirector.java:
171)
at
org.apache.commons.httpclient.HttpClient.executeMethod(HttpClient.java:
397)
at
org.apache.commons.httpclient.HttpClient.executeMethod(HttpClient.java:
323)
at
org
.jets3t
.service
.impl.rest.httpclient.RestS3Service.performRequest(RestS3Service.java:
342)
... 18 more
[WARN ] [12 Jul 2009 00:53:49.361] [Retry]
[com.enstratus.provider.api.CloudException: S3 GET connection failed
for '/']
[WARN ] [12 Jul 2009 00:53:51.568] [RestUtils] [Retried connection 6
times, which exceeds the maximum retry count of 5]
[ERROR] [12 Jul 2009 00:53:51.568] [AmazonS3]
[org.jets3t.service.S3ServiceException: S3 GET connection failed for
'/']
org.jets3t.service.S3ServiceException: S3 GET connection failed for '/'
at
org
.jets3t
.service
.impl.rest.httpclient.RestS3Service.performRequest(RestS3Service.java:
516)
at
org
.jets3t
.service
.impl.rest.httpclient.RestS3Service.performRestGet(RestS3Service.java:
752)
at
org
.jets3t
.service
.impl
.rest
.httpclient.RestS3Service.getBucketLocationImpl(RestS3Service.java:1635)
at org.jets3t.service.S3Service.getBucketLocation(S3Service.java:2316)
at com.enstratus.provider.aws.AmazonS3.listFiles(AmazonS3.java:407)
at
com
.enstratus
.provisioning.storage.StorageFactory.checkState(StorageFactory.java:55)
at com.enstratus.monitor.CloudMonitor$1.call(CloudMonitor.java:277)
at com.enstratus.monitor.CloudMonitor$1.call(CloudMonitor.java:276)
at com.enstratus.util.Retry.retry(Retry.java:32)
at com.enstratus.util.Retry.retry(Retry.java:19)
at com.enstratus.monitor.CloudMonitor.checkStatus(CloudMonitor.java:
282)
at com.enstratus.monitor.CloudMonitor.access$000(CloudMonitor.java:41)
at com.enstratus.monitor.CloudMonitor$2$1.run(CloudMonitor.java:468)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:
441)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at java.util.concurrent.ThreadPoolExecutor
$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor
$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:619)
Caused by: java.net.SocketException: Too many open files
at java.net.Socket.createImpl(Socket.java:388)
at java.net.Socket.getImpl(Socket.java:451)
at java.net.Socket.bind(Socket.java:565)
at
com
.sun
.net.ssl.internal.ssl.BaseSSLSocketImpl.bind(BaseSSLSocketImpl.java:95)
at com.sun.net.ssl.internal.ssl.SSLSocketImpl.bind(SSLSocketImpl.java:
45)
at sun.reflect.GeneratedMethodAccessor31.invoke(Unknown Source)
at
sun
.reflect
.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:
25)
at java.lang.reflect.Method.invoke(Method.java:597)
at
org
.apache
.commons
.httpclient
.protocol
.ReflectionSocketFactory.createSocket(ReflectionSocketFactory.java:139)
at
org
.apache
.commons
.httpclient
.protocol
.SSLProtocolSocketFactory.createSocket(SSLProtocolSocketFactory.java:
130)
at
org.apache.commons.httpclient.HttpConnection.open(HttpConnection.java:
707)
at org.apache.commons.httpclient.MultiThreadedHttpConnectionManager
$HttpConnectionAdapter.open(MultiThreadedHttpConnectionManager.java:
1361)
at
org
.apache
.commons
.httpclient
.HttpMethodDirector.executeWithRetry(HttpMethodDirector.java:387)
at
org
.apache
.commons
.httpclient.HttpMethodDirector.executeMethod(HttpMethodDirector.java:
171)
at
org.apache.commons.httpclient.HttpClient.executeMethod(HttpClient.java:
397)
at
org.apache.commons.httpclient.HttpClient.executeMethod(HttpClient.java:
323)
at
org
.jets3t
.service
.impl.rest.httpclient.RestS3Service.performRequest(RestS3Service.java:
342)
... 18 more
[WARN ] [12 Jul 2009 00:53:51.571] [Retry]
[com.enstratus.provider.api.CloudException: S3 GET connection failed
for '/']
[WARN ] [12 Jul 2009 00:53:55.587] [RestUtils] [Retried connection 6
times, which exceeds the maximum retry count of 5]
[ERROR] [12 Jul 2009 00:53:55.587] [AmazonS3]
[org.jets3t.service.S3ServiceException: S3 GET connection failed for
'/']
org.jets3t.service.S3ServiceException: S3 GET connection failed for '/'
at
org
.jets3t
.service
.impl.rest.httpclient.RestS3Service.performRequest(RestS3Service.java:
516)
at
org
.jets3t
.service
.impl.rest.httpclient.RestS3Service.performRestGet(RestS3Service.java:
752)
at
org
.jets3t
.service
.impl
.rest.httpclient.RestS3Service.listAllBucketsImpl(RestS3Service.java:
1041)
at org.jets3t.service.S3Service.listAllBuckets(S3Service.java:1348)
at com.enstratus.provider.aws.AmazonS3.listFiles(AmazonS3.java:406)
at
com
.enstratus
.provisioning.storage.StorageFactory.checkState(StorageFactory.java:55)
at com.enstratus.monitor.CloudMonitor$1.call(CloudMonitor.java:277)
at com.enstratus.monitor.CloudMonitor$1.call(CloudMonitor.java:276)
at com.enstratus.util.Retry.retry(Retry.java:32)
at com.enstratus.util.Retry.retry(Retry.java:19)
at com.enstratus.monitor.CloudMonitor.checkStatus(CloudMonitor.java:
282)
at com.enstratus.monitor.CloudMonitor.access$000(CloudMonitor.java:41)
at com.enstratus.monitor.CloudMonitor$2$1.run(CloudMonitor.java:468)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:
441)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at java.util.concurrent.ThreadPoolExecutor
$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor
$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:619)
Caused by: java.net.SocketException: Too many open files
at java.net.Socket.createImpl(Socket.java:388)
at java.net.Socket.getImpl(Socket.java:451)
at java.net.Socket.bind(Socket.java:565)
at
com
.sun
.net.ssl.internal.ssl.BaseSSLSocketImpl.bind(BaseSSLSocketImpl.java:95)
at com.sun.net.ssl.internal.ssl.SSLSocketImpl.bind(SSLSocketImpl.java:
45)
at
com.sun.net.ssl.internal.ssl.SSLSocketImpl.<init>(SSLSocketImpl.java:
392)
at
com
.sun
.net
.ssl
.internal
.ssl.SSLSocketFactoryImpl.createSocket(SSLSocketFactoryImpl.java:123)
at
org
.apache
.commons
.httpclient
.protocol
.SSLProtocolSocketFactory.createSocket(SSLProtocolSocketFactory.java:82)
at
org.apache.commons.httpclient.protocol.ControllerThreadSocketFactory
$1.doit(ControllerThreadSocketFactory.java:91)
at
org.apache.commons.httpclient.protocol.ControllerThreadSocketFactory
$SocketTask.run(ControllerThreadSocketFactory.java:158)
... 1 more
--
George Reese (george.re...@enstratus.com - Twitter: @GeorgeReese)
Chief Technology Officer, enStratus Networks LLC
p: +1.612.355.5272 m: +1.612.991.4446 f: +1.612.338.5041
enStratus - Confidence in the Cloud - http://www.enstratus.com