[ https://issues.apache.org/jira/browse/HBASE-14239?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14701587#comment-14701587 ]
Elliott Clark commented on HBASE-14239: --------------------------------------- RIT list contains: {code} Region 1588230740: hbase:meta,,1.1588230740 state=PENDING_CLOSE, ts=Tue Aug 18 03:48:30 PDT 2015 (22029s ago), server=hbase498.ash1.facebook.com,16020,1439847225385 {code} Then the RPC reader threads are blocked here: {code} Thread 29 (RpcServer.reader=5,bindAddress=hbasectrl023.ash1.facebook.com,port=16000): State: WAITING Blocked count: 13 Waited count: 14 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7dc2455e Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.put(LinkedBlockingQueue.java:350) org.apache.hadoop.hbase.ipc.BalancedQueueRpcExecutor.dispatch(BalancedQueueRpcExecutor.java:77) org.apache.hadoop.hbase.ipc.SimpleRpcScheduler.dispatch(SimpleRpcScheduler.java:197) org.apache.hadoop.hbase.ipc.RpcServer$Connection.processRequest(RpcServer.java:1856) org.apache.hadoop.hbase.ipc.RpcServer$Connection.processOneRpc(RpcServer.java:1753) org.apache.hadoop.hbase.ipc.RpcServer$Connection.process(RpcServer.java:1612) org.apache.hadoop.hbase.ipc.RpcServer$Connection.readAndProcess(RpcServer.java:1592) org.apache.hadoop.hbase.ipc.RpcServer$Listener.doRead(RpcServer.java:856) org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:641) org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:617) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) java.lang.Thread.run(Thread.java:745) {code} Priority handlers are all stuck here: {code} Thread 69 (PriorityRpcServer.handler=0,queue=0,port=16000): State: TIMED_WAITING Blocked count: 515 Waited count: 215347 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.client.AsyncProcess$AsyncRequestFutureImpl.waitUntilDone(AsyncProcess.java:1573) org.apache.hadoop.hbase.client.AsyncProcess$AsyncRequestFutureImpl.waitUntilDone(AsyncProcess.java:1543) org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.processBatchCallback(ConnectionManager.java:2231) org.apache.hadoop.hbase.util.MultiHConnection.processBatchCallback(MultiHConnection.java:124) org.apache.hadoop.hbase.master.RegionStateStore.updateRegionState(RegionStateStore.java:244) org.apache.hadoop.hbase.master.RegionStates.updateRegionState(RegionStates.java:1111) org.apache.hadoop.hbase.master.RegionStates.regionOnline(RegionStates.java:449) org.apache.hadoop.hbase.master.AssignmentManager.regionOnline(AssignmentManager.java:1460) org.apache.hadoop.hbase.master.AssignmentManager.onRegionOpen(AssignmentManager.java:3634) org.apache.hadoop.hbase.master.AssignmentManager.onRegionTransition(AssignmentManager.java:4263) org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1339) org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:8623) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2117) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:106) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) java.lang.Thread.run(Thread.java:745) {code} Assignment manager is here: {code} Thread 90156 (AM.-pool1-t403): State: TIMED_WAITING Blocked count: 302 Waited count: 206355 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.client.AsyncProcess$AsyncRequestFutureImpl.waitUntilDone(AsyncProcess.java:1573) org.apache.hadoop.hbase.client.AsyncProcess$AsyncRequestFutureImpl.waitUntilDone(AsyncProcess.java:1543) org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.processBatchCallback(ConnectionManager.java:2231) org.apache.hadoop.hbase.util.MultiHConnection.processBatchCallback(MultiHConnection.java:124) org.apache.hadoop.hbase.master.RegionStateStore.updateRegionState(RegionStateStore.java:244) org.apache.hadoop.hbase.master.RegionStates.updateRegionState(RegionStates.java:1111) org.apache.hadoop.hbase.master.RegionStates.updateRegionState(RegionStates.java:427) org.apache.hadoop.hbase.master.RegionStates.updateRegionState(RegionStates.java:385) org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:2264) org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1587) org.apache.hadoop.hbase.master.AssignCallable.call(AssignCallable.java:48) java.util.concurrent.FutureTask.run(FutureTask.java:266) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) java.lang.Thread.run(Thread.java:745) {code} > Branch-1.2 AM can get stuck when meta moves > ------------------------------------------- > > Key: HBASE-14239 > URL: https://issues.apache.org/jira/browse/HBASE-14239 > Project: HBase > Issue Type: Sub-task > Components: Region Assignment > Reporter: Elliott Clark > Fix For: 1.2.0 > > -- This message was sent by Atlassian JIRA (v6.3.4#6332)