[
https://issues.apache.org/jira/browse/PHOENIX-3448?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Samarth Jain resolved PHOENIX-3448.
-----------------------------------
Resolution: Fixed
> Possible deadlock because transaction service client is stuck
> -------------------------------------------------------------
>
> Key: PHOENIX-3448
> URL: https://issues.apache.org/jira/browse/PHOENIX-3448
> Project: Phoenix
> Issue Type: Bug
> Reporter: Samarth Jain
>
> I have noticed on my local test run that sometimes IndexExtendedIT isn't
> completing. On taking a thread dump, I am seeing two threads waiting for the
> lock (waiting to lock <0x0000000788e69f50>) as follows:
> {code}
> "StoreFileOpenerThread-0-1" daemon prio=5 tid=0x00007fc796e43800 nid=0x2631f
> waiting for monitor entry [0x0000000128f62000]
> java.lang.Thread.State: BLOCKED (on object monitor)
> at
> org.apache.phoenix.query.ConnectionQueryServicesImpl$13.call(ConnectionQueryServicesImpl.java:2369)
> - waiting to lock <0x0000000788e69f50> (a
> org.apache.phoenix.query.ConnectionQueryServicesImpl)
> at
> org.apache.phoenix.query.ConnectionQueryServicesImpl$13.call(ConnectionQueryServicesImpl.java:2358)
> at
> org.apache.phoenix.util.PhoenixContextExecutor.call(PhoenixContextExecutor.java:76)
> at
> org.apache.phoenix.query.ConnectionQueryServicesImpl.init(ConnectionQueryServicesImpl.java:2358)
> at
> org.apache.phoenix.jdbc.PhoenixDriver.getConnectionQueryServices(PhoenixDriver.java:232)
> at
> org.apache.phoenix.jdbc.PhoenixEmbeddedDriver.createConnection(PhoenixEmbeddedDriver.java:147)
> at org.apache.phoenix.jdbc.PhoenixDriver.connect(PhoenixDriver.java:202)
> at java.sql.DriverManager.getConnection(DriverManager.java:571)
> at java.sql.DriverManager.getConnection(DriverManager.java:187)
> at org.apache.phoenix.util.QueryUtil.getConnection(QueryUtil.java:341)
> at
> org.apache.phoenix.util.QueryUtil.getConnectionOnServer(QueryUtil.java:328)
> at
> org.apache.phoenix.util.QueryUtil.getConnectionOnServer(QueryUtil.java:318)
> at
> org.apache.hadoop.hbase.regionserver.IndexHalfStoreFileReaderGenerator.preStoreFileReaderOpen(IndexHalfStoreFileReaderGenerator.java:144)
> at
> org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$64.call(RegionCoprocessorHost.java:1545)
> at
> org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$RegionOperation.call(RegionCoprocessorHost.java:1656)
> at
> org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.execOperation(RegionCoprocessorHost.java:1733)
> at
> org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.execOperationWithResult(RegionCoprocessorHost.java:1695)
> at
> org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preStoreFileReaderOpen(RegionCoprocessorHost.java:1540)
> at
> org.apache.hadoop.hbase.regionserver.StoreFileInfo.open(StoreFileInfo.java:205)
> at
> org.apache.hadoop.hbase.regionserver.StoreFile.open(StoreFile.java:380)
> at
> org.apache.hadoop.hbase.regionserver.StoreFile.createReader(StoreFile.java:481)
> at
> org.apache.hadoop.hbase.regionserver.StoreFile.createReader(StoreFile.java:471)
> at
> org.apache.hadoop.hbase.regionserver.HStore.createStoreFileAndReader(HStore.java:571)
> at
> org.apache.hadoop.hbase.regionserver.HStore.access$000(HStore.java:123)
> at org.apache.hadoop.hbase.regionserver.HStore$1.call(HStore.java:513)
> at org.apache.hadoop.hbase.regionserver.HStore$1.call(HStore.java:510)
> at java.util.concurrent.FutureTask.run(FutureTask.java:262)
> at
> java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
> at java.util.concurrent.FutureTask.run(FutureTask.java:262)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
> at java.lang.Thread.run(Thread.java:745)
> {code}
> The thread which has the lock is stuck because starting the transaction
> client service hasn't returned. The stacktrace for that thread looks like
> this:
> {code}
> "StoreFileOpenerThread-0-1" daemon prio=5 tid=0x00007fc797088000 nid=0x29717
> waiting on condition [0x00000001295e1000]
> java.lang.Thread.State: WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for <0x0000000788f1bb58> (a
> com.google.common.util.concurrent.AbstractFuture$Sync)
> at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:834)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:994)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1303)
> at
> com.google.common.util.concurrent.AbstractFuture$Sync.get(AbstractFuture.java:280)
> at
> com.google.common.util.concurrent.AbstractFuture.get(AbstractFuture.java:116)
> at
> com.google.common.util.concurrent.Uninterruptibles.getUninterruptibly(Uninterruptibles.java:132)
> at
> com.google.common.util.concurrent.Futures.getUnchecked(Futures.java:999)
> at
> org.apache.twill.zookeeper.ForwardingZKClientService.startAndWait(ForwardingZKClientService.java:51)
> at
> org.apache.phoenix.query.ConnectionQueryServicesImpl.initTxServiceClient(ConnectionQueryServicesImpl.java:384)
> at
> org.apache.phoenix.query.ConnectionQueryServicesImpl.openConnection(ConnectionQueryServicesImpl.java:398)
> at
> org.apache.phoenix.query.ConnectionQueryServicesImpl.access$300(ConnectionQueryServicesImpl.java:234)
> at
> org.apache.phoenix.query.ConnectionQueryServicesImpl$13.call(ConnectionQueryServicesImpl.java:2380)
> - locked <0x0000000788e69f50> (a
> org.apache.phoenix.query.ConnectionQueryServicesImpl)
> at
> org.apache.phoenix.query.ConnectionQueryServicesImpl$13.call(ConnectionQueryServicesImpl.java:2358)
> at
> org.apache.phoenix.util.PhoenixContextExecutor.call(PhoenixContextExecutor.java:76)
> at
> org.apache.phoenix.query.ConnectionQueryServicesImpl.init(ConnectionQueryServicesImpl.java:2358)
> at
> org.apache.phoenix.jdbc.PhoenixDriver.getConnectionQueryServices(PhoenixDriver.java:232)
> at
> org.apache.phoenix.jdbc.PhoenixEmbeddedDriver.createConnection(PhoenixEmbeddedDriver.java:147)
> at org.apache.phoenix.jdbc.PhoenixDriver.connect(PhoenixDriver.java:202)
> at java.sql.DriverManager.getConnection(DriverManager.java:571)
> at java.sql.DriverManager.getConnection(DriverManager.java:187)
> at org.apache.phoenix.util.QueryUtil.getConnection(QueryUtil.java:341)
> at
> org.apache.phoenix.util.QueryUtil.getConnectionOnServer(QueryUtil.java:328)
> at
> org.apache.phoenix.util.QueryUtil.getConnectionOnServer(QueryUtil.java:318)
> at
> org.apache.hadoop.hbase.regionserver.IndexHalfStoreFileReaderGenerator.preStoreFileReaderOpen(IndexHalfStoreFileReaderGenerator.java:144)
> at
> org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$64.call(RegionCoprocessorHost.java:1545)
> at
> org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$RegionOperation.call(RegionCoprocessorHost.java:1656)
> at
> org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.execOperation(RegionCoprocessorHost.java:1733)
> at
> org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.execOperationWithResult(RegionCoprocessorHost.java:1695)
> at
> org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preStoreFileReaderOpen(RegionCoprocessorHost.java:1540)
> at
> org.apache.hadoop.hbase.regionserver.StoreFileInfo.open(StoreFileInfo.java:205)
> at
> org.apache.hadoop.hbase.regionserver.StoreFile.open(StoreFile.java:380)
> at
> org.apache.hadoop.hbase.regionserver.StoreFile.createReader(StoreFile.java:481)
> at
> org.apache.hadoop.hbase.regionserver.StoreFile.createReader(StoreFile.java:471)
> at
> org.apache.hadoop.hbase.regionserver.HStore.createStoreFileAndReader(HStore.java:571)
> at
> org.apache.hadoop.hbase.regionserver.HStore.access$000(HStore.java:123)
> at org.apache.hadoop.hbase.regionserver.HStore$1.call(HStore.java:513)
> at org.apache.hadoop.hbase.regionserver.HStore$1.call(HStore.java:510)
> at java.util.concurrent.FutureTask.run(FutureTask.java:262)
> at
> java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
> at java.util.concurrent.FutureTask.run(FutureTask.java:262)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
> at java.lang.Thread.run(Thread.java:745)
> {code}
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
