[ https://issues.apache.org/jira/browse/HBASE-19940?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16354604#comment-16354604 ]
stack commented on HBASE-19940: ------------------------------- Ok. Got my thread dumps. RS is here: java.lang.Thread.sleep(Native Method) org.apache.hadoop.hbase.regionserver.HRegionServer.sleep(HRegionServer.java:1419) org.apache.hadoop.hbase.regionserver.HRegionServer.waitOnAllRegionsToClose(HRegionServer.java:1405) org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:1097) org.apache.hadoop.hbase.MiniHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(MiniHBaseCluster.java:187) org.apache.hadoop.hbase.MiniHBaseCluster$MiniHBaseClusterRegionServer.access$000(MiniHBaseCluster.java:133) org.apache.hadoop.hbase.MiniHBaseCluster$MiniHBaseClusterRegionServer$1.run(MiniHBaseCluster.java:171) java.security.AccessController.doPrivileged(Native Method) javax.security.auth.Subject.doAs(Subject.java:360) org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1726) org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:307) org.apache.hadoop.hbase.MiniHBaseCluster$MiniHBaseClusterRegionServer.run(MiniHBaseCluster.java:168) java.lang.Thread.run(Thread.java:748) Regions are here: org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:304) Thread 462 (RpcServer.priority.FPBQ.Fifo.handler=5,queue=0,port=44612): State: TIMED_WAITING Blocked count: 7 Waited count: 220 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:133) org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.blockOnSync(AbstractFSWAL.java:718) org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.sync(AsyncFSWAL.java:605) org.apache.hadoop.hbase.regionserver.HRegion.sync(HRegion.java:8255) org.apache.hadoop.hbase.regionserver.HRegion.doWALAppend(HRegion.java:7630) org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutate(HRegion.java:3865) org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:3810) org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:3741) org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:977) org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:896) org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2593) org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:41560) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:406) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:130) org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:324) org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:304) Thread 461 (RpcServer.priority.FPBQ.Fifo.handler=4,queue=0,port=44612): And here.... Thread 830 (RS_CLOSE_META-regionserver/asf906:0-0): State: WAITING Blocked count: 1 Waited count: 1 Waiting on java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@60664e9b Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870) java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1199) java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock.lock(ReentrantReadWriteLock.java:943) org.apache.hadoop.hbase.regionserver.HRegion.doClose(HRegion.java:1503) org.apache.hadoop.hbase.regionserver.HRegion.close(HRegion.java:1437) org.apache.hadoop.hbase.regionserver.handler.CloseRegionHandler.process(CloseRegionHandler.java:104) org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:748) A version of HBASE-19929 ? > TestMetaShutdownHandler flakey > ------------------------------ > > Key: HBASE-19940 > URL: https://issues.apache.org/jira/browse/HBASE-19940 > Project: HBase > Issue Type: Sub-task > Reporter: stack > Priority: Major > Fix For: 2.0.0-beta-2 > > Attachments: HBASE-19940.master.001.patch, > HBASE-19940.master.002.patch > > > Fails 13% of the time. One of the RS won't go down. It has an errant thread > running. Not sure what. -- This message was sent by Atlassian JIRA (v7.6.3#76005)