[
https://issues.apache.org/jira/browse/HTTPASYNC-153?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16912151#comment-16912151
]
Max Rozhkov commented on HTTPASYNC-153:
---------------------------------------
It seems you are right. It's not classic deadlock. At the same time, all the
blocked and waiting working threads don't unlock. CPU is utilized much more
than usually, the load raises from 40% to 80%.
I have no idea how to reproduce it with a snippet. The same code worked well
for apparently 6 months. But lately it happens pretty often. How could I get
additional information for you?
Thank you for investigating the problem!
Max
> Deadlock when sending new request
> ---------------------------------
>
> Key: HTTPASYNC-153
> URL: https://issues.apache.org/jira/browse/HTTPASYNC-153
> Project: HttpComponents HttpAsyncClient
> Issue Type: Bug
> Affects Versions: 4.1.4
> Environment: CentOS Linux release 7.5.1804
> Java 11
> Reporter: Max Rozhkov
> Priority: Blocker
> Attachments: thread_print-190821-082533
>
>
> We faced a situation that async http requests are blocked at the moment they
> are being made. This blocks work thread which make all the threads in the
> pool blocked.
> At the same time, the thread which closes idle connections is also blocked.
> Please fix the problem. Otherwise we have to move to another http client asap.
> Typical stack traces are below:
>
> java.lang.Thread.State: BLOCKED (on object monitor)
> at org.apache.http.nio.reactor.ssl.SSLIOSession.close(SSLIOSession.java:605)
> - waiting to lock <0x00000011c1caa2a0> (a
> org.apache.http.nio.reactor.ssl.SSLIOSession)
> at
> org.apache.http.impl.nio.NHttpConnectionBase.close(NHttpConnectionBase.java:511)
> at
> org.apache.http.impl.nio.conn.CPoolEntry.closeConnection(CPoolEntry.java:75)
> at org.apache.http.impl.nio.conn.CPoolEntry.close(CPoolEntry.java:101)
> at
> org.apache.http.nio.pool.AbstractNIOConnPool$4.process(AbstractNIOConnPool.java:846)
> at
> org.apache.http.nio.pool.AbstractNIOConnPool.enumAvailable(AbstractNIOConnPool.java:775)
> at
> org.apache.http.nio.pool.AbstractNIOConnPool.closeIdle(AbstractNIOConnPool.java:841)
> at
> org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager.closeIdleConnections(PoolingNHttpClientConnectionManager.java:493)
> at
> net.thumbtack.ssp.requester.AsyncHttpClient.lambda$monitorConnections$0(AsyncHttpClient.java:67)
> at
> net.thumbtack.ssp.requester.AsyncHttpClient$$Lambda$311/0x00000017c2616440.run(Unknown
> Source)
> at
> java.util.concurrent.Executors$RunnableAdapter.call([email protected]/Executors.java:515)
> at
> java.util.concurrent.FutureTask.runAndReset([email protected]/FutureTask.java:305)
> at
> java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run([email protected]/ScheduledThreadPoolExecutor.java:305)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker([email protected]/ThreadPoolExecutor.java:1128)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run([email protected]/ThreadPoolExecutor.java:628)
> at java.lang.Thread.run([email protected]/Thread.java:834)
>
> java.lang.Thread.State: WAITING (parking)
> at jdk.internal.misc.Unsafe.park([email protected]/Native Method)
> - parking to wait for <0x00000011310a4150> (a
> java.util.concurrent.locks.ReentrantLock$NonfairSync)
> at
> java.util.concurrent.locks.LockSupport.park([email protected]/LockSupport.java:194)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt([email protected]/AbstractQueuedSynchronizer.java:885)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued([email protected]/AbstractQueuedSynchronizer.java:917)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire([email protected]/AbstractQueuedSynchronizer.java:1240)
> at
> java.util.concurrent.locks.ReentrantLock.lock([email protected]/ReentrantLock.java:267)
> at
> org.apache.http.nio.pool.AbstractNIOConnPool.lease(AbstractNIOConnPool.java:278)
> at
> org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager.requestConnection(PoolingNHttpClientConnectionManager.java:295)
> at
> org.apache.http.impl.nio.client.AbstractClientExchangeHandler.requestConnection(AbstractClientExchangeHandler.java:377)
> at
> org.apache.http.impl.nio.client.DefaultClientExchangeHandlerImpl.start(DefaultClientExchangeHandlerImpl.java:129)
> at
> org.apache.http.impl.nio.client.InternalHttpAsyncClient.execute(InternalHttpAsyncClient.java:141)
> at
> org.apache.http.impl.nio.client.CloseableHttpAsyncClient.execute(CloseableHttpAsyncClient.java:75)
> at
> org.apache.http.impl.nio.client.CloseableHttpAsyncClient.execute(CloseableHttpAsyncClient.java:108)
> at
> net.thumbtack.ssp.requester.AsyncHttpClient.execute(AsyncHttpClient.java:96)
> at
> net.thumbtack.ssp.requester.SelfHealingAsyncHttpClient.execute(SelfHealingAsyncHttpClient.java:45)
> at
> net.thumbtack.ssp.requester.AsyncHttpRequester.get(AsyncHttpRequester.java:125)
> at net.thumbtack.ssp.requester.HttpRequester.get(HttpRequester.java:38)
> at
> net.thumbtack.ssp.impression.tracking.util.PartnerHttpNotifier.doWinNotification(PartnerHttpNotifier.java:138)
> at
> net.thumbtack.ssp.impression.tracking.util.PartnerHttpNotifier.notifyPartner(PartnerHttpNotifier.java:65)
> at
> net.thumbtack.adtech.notification.NotificationEventLoggerFactory.lambda$notifyPartner$1(NotificationEventLoggerFactory.java:43)
> at
> net.thumbtack.adtech.notification.NotificationEventLoggerFactory$$Lambda$1897/0x00000017c2a13040.accept(Unknown
> Source)
> at java.util.ArrayList.forEach([email protected]/ArrayList.java:1540)
> at
> net.thumbtack.adtech.notification.NotificationEventLoggerFactory.notifyPartner(NotificationEventLoggerFactory.java:43)
> at
> net.thumbtack.adtech.notification.NotificationEventLoggerFactory.lambda$getImpressionsLogger$0(NotificationEventLoggerFactory.java:39)
> at
> net.thumbtack.adtech.notification.NotificationEventLoggerFactory$$Lambda$324/0x00000017c2612c40.log(Unknown
> Source)
> at
> net.thumbtack.adtech.event.CompositeAsyncEventLoggerFactory.lambda$runLoggersAsync$8(CompositeAsyncEventLoggerFactory.java:109)
> at
> net.thumbtack.adtech.event.CompositeAsyncEventLoggerFactory$$Lambda$1889/0x00000017c2a15040.accept(Unknown
> Source)
> at java.util.ArrayList.forEach([email protected]/ArrayList.java:1540)
> at
> net.thumbtack.adtech.event.CompositeAsyncEventLoggerFactory.lambda$runLoggersAsync$9(CompositeAsyncEventLoggerFactory.java:107)
> at
> net.thumbtack.adtech.event.CompositeAsyncEventLoggerFactory$$Lambda$1881/0x00000017c2a17040.run(Unknown
> Source)
> at
> java.util.concurrent.CompletableFuture$AsyncRun.run([email protected]/CompletableFuture.java:1736)
> at
> java.util.concurrent.Executors$RunnableAdapter.call([email protected]/Executors.java:515)
> at java.util.concurrent.FutureTask.run([email protected]/FutureTask.java:264)
> at
> java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run([email protected]/ScheduledThreadPoolExecutor.java:304)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker([email protected]/ThreadPoolExecutor.java:1128)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run([email protected]/ThreadPoolExecutor.java:628)
> at java.lang.Thread.run([email protected]/Thread.java:834)
>
> java.lang.Thread.State: BLOCKED (on object monitor)
> at
> org.apache.http.nio.reactor.ssl.SSLIOSession.setEvent(SSLIOSession.java:672)
> - waiting to lock <0x00000011c1c7d080> (a
> org.apache.http.nio.reactor.ssl.SSLIOSession)
> at
> org.apache.http.impl.nio.NHttpConnectionBase.requestOutput(NHttpConnectionBase.java:358)
> at
> org.apache.http.impl.nio.conn.CPoolProxy.requestOutput(CPoolProxy.java:120)
> at
> org.apache.http.impl.nio.client.AbstractClientExchangeHandler.connectionAllocated(AbstractClientExchangeHandler.java:334)
> - locked <0x00000011c1d046c0> (a org.apache.http.impl.nio.SessionHttpContext)
> at
> org.apache.http.impl.nio.client.AbstractClientExchangeHandler.access$000(AbstractClientExchangeHandler.java:62)
> at
> org.apache.http.impl.nio.client.AbstractClientExchangeHandler$1.completed(AbstractClientExchangeHandler.java:387)
> at
> org.apache.http.impl.nio.client.AbstractClientExchangeHandler$1.completed(AbstractClientExchangeHandler.java:383)
> at org.apache.http.concurrent.BasicFuture.completed(BasicFuture.java:122)
> at
> org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager$1.completed(PoolingNHttpClientConnectionManager.java:306)
> at
> org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager$1.completed(PoolingNHttpClientConnectionManager.java:297)
> at org.apache.http.concurrent.BasicFuture.completed(BasicFuture.java:122)
> at
> org.apache.http.nio.pool.AbstractNIOConnPool.fireCallbacks(AbstractNIOConnPool.java:506)
> at
> org.apache.http.nio.pool.AbstractNIOConnPool.lease(AbstractNIOConnPool.java:290)
> at
> org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager.requestConnection(PoolingNHttpClientConnectionManager.java:295)
> at
> org.apache.http.impl.nio.client.AbstractClientExchangeHandler.requestConnection(AbstractClientExchangeHandler.java:377)
> at
> org.apache.http.impl.nio.client.DefaultClientExchangeHandlerImpl.start(DefaultClientExchangeHandlerImpl.java:129)
> at
> org.apache.http.impl.nio.client.InternalHttpAsyncClient.execute(InternalHttpAsyncClient.java:141)
> at
> org.apache.http.impl.nio.client.CloseableHttpAsyncClient.execute(CloseableHttpAsyncClient.java:75)
> at
> org.apache.http.impl.nio.client.CloseableHttpAsyncClient.execute(CloseableHttpAsyncClient.java:108)
> at
> net.thumbtack.ssp.requester.AsyncHttpClient.execute(AsyncHttpClient.java:96)
> at
> net.thumbtack.ssp.requester.SelfHealingAsyncHttpClient.execute(SelfHealingAsyncHttpClient.java:45)
> at
> net.thumbtack.ssp.requester.AsyncHttpRequester.get(AsyncHttpRequester.java:125)
> at net.thumbtack.ssp.requester.HttpRequester.get(HttpRequester.java:38)
> at
> net.thumbtack.ssp.impression.tracking.util.PartnerHttpNotifier.doWinNotification(PartnerHttpNotifier.java:138)
> at
> net.thumbtack.ssp.impression.tracking.util.PartnerHttpNotifier.notifyPartner(PartnerHttpNotifier.java:59)
> at
> net.thumbtack.adtech.notification.NotificationEventLoggerFactory.lambda$notifyPartner$1(NotificationEventLoggerFactory.java:43)
> at
> net.thumbtack.adtech.notification.NotificationEventLoggerFactory$$Lambda$1897/0x00000017c2a13040.accept(Unknown
> Source)
> at
> java.util.Collections$SingletonList.forEach([email protected]/Collections.java:4854)
> at
> net.thumbtack.adtech.notification.NotificationEventLoggerFactory.notifyPartner(NotificationEventLoggerFactory.java:43)
> at
> net.thumbtack.adtech.notification.NotificationEventLoggerFactory$$Lambda$323/0x00000017c2612840.log(Unknown
> Source)
> at
> net.thumbtack.adtech.event.CompositeAsyncEventLoggerFactory.lambda$runLoggersAsync$8(CompositeAsyncEventLoggerFactory.java:109)
> at
> net.thumbtack.adtech.event.CompositeAsyncEventLoggerFactory$$Lambda$1889/0x00000017c2a15040.accept(Unknown
> Source)
> at java.util.ArrayList.forEach([email protected]/ArrayList.java:1540)
> at
> net.thumbtack.adtech.event.CompositeAsyncEventLoggerFactory.lambda$runLoggersAsync$9(CompositeAsyncEventLoggerFactory.java:107)
> at
> net.thumbtack.adtech.event.CompositeAsyncEventLoggerFactory$$Lambda$1881/0x00000017c2a17040.run(Unknown
> Source)
> at
> java.util.concurrent.CompletableFuture$AsyncRun.run([email protected]/CompletableFuture.java:1736)
> at
> java.util.concurrent.Executors$RunnableAdapter.call([email protected]/Executors.java:515)
> at java.util.concurrent.FutureTask.run([email protected]/FutureTask.java:264)
> at
> java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run([email protected]/ScheduledThreadPoolExecutor.java:304)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker([email protected]/ThreadPoolExecutor.java:1128)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run([email protected]/ThreadPoolExecutor.java:628)
> at java.lang.Thread.run([email protected]/Thread.java:834)
--
This message was sent by Atlassian Jira
(v8.3.2#803003)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
