[ https://issues.apache.org/jira/browse/PHOENIX-4474?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Ankit Singhal resolved PHOENIX-4474. ------------------------------------ Resolution: Fixed With latest fixes, these tests are also not hanging now. > Found some hanging tests > ------------------------ > > Key: PHOENIX-4474 > URL: https://issues.apache.org/jira/browse/PHOENIX-4474 > Project: Phoenix > Issue Type: Sub-task > Reporter: Ankit Singhal > Labels: HBase-2.0 > Fix For: 5.0.0 > > > * ExplainPlanWithStatsDisabledIT > * ConcurrentMutationsIT > * NumericArithmeticIT > * AggregateQueryIT > AggregateQueryIT > {code} > Mon Dec 18 23:49:20 IST 2017, > RpcRetryingCaller{globalStartTime=1513621155916, pause=100, maxAttempts=7}, > java.net.ConnectException: Call to /192.168.1.3:56675 failed on connection > exception: > org.apache.hadoop.hbase.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: > Connection refused: /192.168.1.3:56675 > at > org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:145) > at org.apache.hadoop.hbase.client.HTable.get(HTable.java:388) > at org.apache.hadoop.hbase.client.HTable.get(HTable.java:362) > at > org.apache.hadoop.hbase.MetaTableAccessor.getTableState(MetaTableAccessor.java:1118) > at > org.apache.hadoop.hbase.master.TableStateManager.readMetaState(TableStateManager.java:190) > at > org.apache.hadoop.hbase.master.TableStateManager.isTablePresent(TableStateManager.java:147) > at > org.apache.hadoop.hbase.master.HMaster.getTableDescriptors(HMaster.java:3135) > at > org.apache.hadoop.hbase.master.HMaster.listTableDescriptors(HMaster.java:3079) > at > org.apache.hadoop.hbase.master.MasterRpcServices.getTableDescriptors(MasterRpcServices.java:999) > at > org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) > at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:403) > at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:130) > at > org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:325) > at > org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:305) > Caused by: java.net.ConnectException: Call to /192.168.1.3:56675 failed on > connection exception: > org.apache.hadoop.hbase.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: > Connection refused: /192.168.1.3:56675 > at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:165) > at > org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:390) > at > org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:95) > at > org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:410) > at > org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:406) > at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:103) > at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:118) > at > org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:92) > at > org.apache.hadoop.hbase.shaded.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:329) > at > org.apache.hadoop.hbase.shaded.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:315) > at > org.apache.hadoop.hbase.shaded.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:307) > at > org.apache.hadoop.hbase.shaded.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1352) > at > org.apache.hadoop.hbase.shaded.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:329) > at > org.apache.hadoop.hbase.shaded.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:315) > at > org.apache.hadoop.hbase.shaded.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:920) > at > org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:179) > at > org.apache.hadoop.hbase.ipc.NettyRpcConnection.access$500(NettyRpcConnection.java:71) > at > org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:269) > at > org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:263) > at > org.apache.hadoop.hbase.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:507) > at > org.apache.hadoop.hbase.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:500) > at > org.apache.hadoop.hbase.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:479) > at > org.apache.hadoop.hbase.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:420) > at > org.apache.hadoop.hbase.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:122) > at > org.apache.hadoop.hbase.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:327) > at > org.apache.hadoop.hbase.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:343) > at > org.apache.hadoop.hbase.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:632) > at > org.apache.hadoop.hbase.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:579) > at > org.apache.hadoop.hbase.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:496) > at > org.apache.hadoop.hbase.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:458) > at > org.apache.hadoop.hbase.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858) > at > org.apache.hadoop.hbase.shaded.io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:138) > at java.lang.Thread.run(Thread.java:748) > Caused by: > org.apache.hadoop.hbase.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: > Connection refused: /192.168.1.3:56675 > at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) > at > sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717) > at > org.apache.hadoop.hbase.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:352) > at > org.apache.hadoop.hbase.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:340) > ... 7 more > Caused by: java.net.ConnectException: Connection refused > ... 11 more > [ERROR] Tests run: 7, Failures: 0, Errors: 1, Skipped: 0, Time elapsed: > 705.751 s <<< FAILURE! - in org.apache.phoenix.end2end.NativeHBaseTypesIT > [ERROR] testRangeQuery1(org.apache.phoenix.end2end.NativeHBaseTypesIT) Time > elapsed: 610.63 s <<< ERROR! > org.apache.phoenix.exception.PhoenixIOException: > java.util.concurrent.TimeoutException: The procedure 37 is still running > at > org.apache.phoenix.end2end.NativeHBaseTypesIT.initTableValues(NativeHBaseTypesIT.java:154) > at > org.apache.phoenix.end2end.NativeHBaseTypesIT.testRangeQuery1(NativeHBaseTypesIT.java:162) > Caused by: org.apache.hadoop.hbase.exceptions.TimeoutIOException: > java.util.concurrent.TimeoutException: The procedure 37 is still running > at > org.apache.phoenix.end2end.NativeHBaseTypesIT.initTableValues(NativeHBaseTypesIT.java:154) > at > org.apache.phoenix.end2end.NativeHBaseTypesIT.testRangeQuery1(NativeHBaseTypesIT.java:162) > Caused by: java.util.concurrent.TimeoutException: The procedure 37 is still > running > at > org.apache.phoenix.end2end.NativeHBaseTypesIT.initTableValues(NativeHBaseTypesIT.java:154) > at > org.apache.phoenix.end2end.NativeHBaseTypesIT.testRangeQuery1(NativeHBaseTypesIT.java:162) > {code} > NumericArithmeticIT > {code} > java.io.IOException: java.util.concurrent.ExecutionException: > java.io.IOException: The specified region already exists on disk: > hdfs://localhost:57532/user/asinghal/test-data/0c88fe5d-8cbc-495c-9668-04ab6847b35f/.tmp/data/default/T000365/d92d9658645e600021fc351c4af8ba64 > at > org.apache.hadoop.hbase.util.ModifyRegionUtils.createRegions(ModifyRegionUtils.java:159) > at > org.apache.hadoop.hbase.util.ModifyRegionUtils.createRegions(ModifyRegionUtils.java:116) > at > org.apache.hadoop.hbase.master.procedure.CreateTableProcedure$1.createHdfsRegions(CreateTableProcedure.java:290) > at > org.apache.hadoop.hbase.master.procedure.CreateTableProcedure.createFsLayout(CreateTableProcedure.java:310) > at > org.apache.hadoop.hbase.master.procedure.CreateTableProcedure.createFsLayout(CreateTableProcedure.java:283) > at > org.apache.hadoop.hbase.master.procedure.CreateTableProcedure.executeFromState(CreateTableProcedure.java:100) > at > org.apache.hadoop.hbase.master.procedure.CreateTableProcedure.executeFromState(CreateTableProcedure.java:52) > at > org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:182) > at > org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:845) > at > org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1455) > at > org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1224) > at > org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$800(ProcedureExecutor.java:77) > at > org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1734) > Caused by: java.util.concurrent.ExecutionException: java.io.IOException: The > specified region already exists on disk: > hdfs://localhost:57532/user/asinghal/test-data/0c88fe5d-8cbc-495c-9668-04ab6847b35f/.tmp/data/default/T000365/d92d9658645e600021fc351c4af8ba64 > at java.util.concurrent.FutureTask.report(FutureTask.java:122) > at java.util.concurrent.FutureTask.get(FutureTask.java:192) > at > org.apache.hadoop.hbase.util.ModifyRegionUtils.createRegions(ModifyRegionUtils.java:153) > ... 12 more > Caused by: java.io.IOException: The specified region already exists on disk: > hdfs://localhost:57532/user/asinghal/test-data/0c88fe5d-8cbc-495c-9668-04ab6847b35f/.tmp/data/default/T000365/d92d9658645e600021fc351c4af8ba64 > at > org.apache.hadoop.hbase.regionserver.HRegionFileSystem.createRegionOnFileSystem(HRegionFileSystem.java:975) > at > org.apache.hadoop.hbase.regionserver.HRegion.createHRegion(HRegion.java:6776) > at > org.apache.hadoop.hbase.util.ModifyRegionUtils.createRegion(ModifyRegionUtils.java:181) > at > org.apache.hadoop.hbase.util.ModifyRegionUtils$1.call(ModifyRegionUtils.java:146) > at > org.apache.hadoop.hbase.util.ModifyRegionUtils$1.call(ModifyRegionUtils.java:143) > at java.util.concurrent.FutureTask.run(FutureTask.java:266) > at > java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) > at java.util.concurrent.FutureTask.run(FutureTask.java:266) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > at java.lang.Thread.run(Thread.java:748) > 2017-12-19 00:40:17,051 FATAL [ProcExecWrkr-15] > org.apache.hadoop.hbase.procedure2.ProcedureExecutor(1367): CODE-BUG: > Uncaught runtime exception for pid=137, > state=FAILED:CREATE_TABLE_WRITE_FS_LAYOUT, > exception=java.lang.OutOfMemoryError via CODE-BUG: Uncaught runtime > exception: pid=137, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT; > CreateTableProcedure table=T000360:java.lang.OutOfMemoryError: unable to > create new native thread; CreateTableProcedure table=T000360 > java.lang.UnsupportedOperationException: unhandled > state=CREATE_TABLE_WRITE_FS_LAYOUT > at > org.apache.hadoop.hbase.master.procedure.CreateTableProcedure.rollbackState(CreateTableProcedure.java:146) > at > org.apache.hadoop.hbase.master.procedure.CreateTableProcedure.rollbackState(CreateTableProcedure.java:52) > at > org.apache.hadoop.hbase.procedure2.StateMachineProcedure.rollback(StateMachineProcedure.java:201) > at > org.apache.hadoop.hbase.procedure2.Procedure.doRollback(Procedure.java:859) > at > org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1356) > at > org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1312) > at > org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1181) > at > org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$800(ProcedureExecutor.java:77) > at > org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1734) > {code} -- This message was sent by Atlassian JIRA (v6.4.14#64029)