[
https://issues.apache.org/jira/browse/PHOENIX-4715?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Thomas D'Silva updated PHOENIX-4715:
------------------------------------
Description:
I think the 3 test failures in PartialIndexRebuilderIT started happening after
we switched master to HBase 1.4 as part of PHOENIX-4076.
Maybe [~lhofhansl] or [~apurtell] might have some insight
{code:java}
[ERROR] Failures:
[ERROR] PartialIndexRebuilderIT.testConcurrentUpsertsWithRebuild:230 Expected
equality for V1, but null!=11
[ERROR] PartialIndexRebuilderIT.testDeleteAndUpsertAfterFailure:347 Expected
equality for V2, but null!=1
[ERROR] PartialIndexRebuilderIT.testWriteWhileRebuilding:396 Expected equality
for V2, but null!=2
{code}
testDeleteAndUpsertAfterFailure and testWriteWhileRebuilding pass for me
locally just before PHOENIX-4076 was committed.
testConcurrentUpsertsWithRebuild fails with the following exception at the
commit before PHOENIX-4076 .
{code:java}
2018-04-27 16:14:48,049 ERROR
[RpcServer.FifoWFPBQ.default.handler=1,queue=0,port=26069]
org.apache.phoenix.coprocessor.UngroupedAggregateRegionObserver(1089):
IOException during rebuilding:
org.apache.hadoop.hbase.exceptions.TimeoutIOException: Timed out waiting for
lock for row: 80 00 00 01 80 00 00 00
at
org.apache.phoenix.hbase.index.LockManager.lockRow(LockManager.java:96)
at
org.apache.phoenix.hbase.index.Indexer.preBatchMutateWithExceptions(Indexer.java:421)
at
org.apache.phoenix.hbase.index.Indexer.preBatchMutate(Indexer.java:370)
at
org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$35.call(RegionCoprocessorHost.java:1007)
at
org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$RegionOperation.call(RegionCoprocessorHost.java:1673)
at
org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.execOperation(RegionCoprocessorHost.java:1749)
at
org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.execOperation(RegionCoprocessorHost.java:1705)
at
org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preBatchMutate(RegionCoprocessorHost.java:1003)
at
org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3190)
at
org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2976)
at
org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2918)
at
org.apache.phoenix.coprocessor.UngroupedAggregateRegionObserver.rebuildIndices(UngroupedAggregateRegionObserver.java:1074)
at
org.apache.phoenix.coprocessor.UngroupedAggregateRegionObserver.doPostScannerOpen(UngroupedAggregateRegionObserver.java:369)
at
org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.overrideDelegate(BaseScannerRegionObserver.java:245)
at
org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:293)
at
org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:2629)
at
org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:2833)
at
org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:34950)
at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2339)
at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:123)
at
org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:188)
at
org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:168)
2018-04-27 16:14:48,051 DEBUG
[RpcServer.FifoWFPBQ.default.handler=1,queue=0,port=26069]
org.apache.hadoop.hbase.ipc.CallRunner(126):
RpcServer.FifoWFPBQ.default.handler=1,queue=0,port=26069: callId: 1941 service:
ClientService methodName: Scan size: 40 connection: 127.0.0.1:14017
org.apache.hadoop.hbase.UnknownScannerException: Throwing
UnknownScannerException to reset the client scanner state for clients older
than 1.3.
at
org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:2893)
at
org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:34950)
at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2339)
at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:123)
at
org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:188)
at
org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:168)
Caused by: org.apache.hadoop.hbase.exceptions.TimeoutIOException: Timed out
waiting for lock for row: 80 00 00 01 80 00 00 00
at
org.apache.phoenix.hbase.index.LockManager.lockRow(LockManager.java:96)
at
org.apache.phoenix.hbase.index.Indexer.preBatchMutateWithExceptions(Indexer.java:421)
at
org.apache.phoenix.hbase.index.Indexer.preBatchMutate(Indexer.java:370)
at
org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$35.call(RegionCoprocessorHost.java:1007)
at
org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$RegionOperation.call(RegionCoprocessorHost.java:1673)
at
org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.execOperation(RegionCoprocessorHost.java:1749)
at
org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.execOperation(RegionCoprocessorHost.java:1705)
at
org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preBatchMutate(RegionCoprocessorHost.java:1003)
at
org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3190)
at
org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2976)
at
org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2918)
at
org.apache.phoenix.coprocessor.UngroupedAggregateRegionObserver.rebuildIndices(UngroupedAggregateRegionObserver.java:1074)
at
org.apache.phoenix.coprocessor.UngroupedAggregateRegionObserver.doPostScannerOpen(UngroupedAggregateRegionObserver.java:369)
at
org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.overrideDelegate(BaseScannerRegionObserver.java:245)
at
org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:293)
at
org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:2629)
at
org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:2833)
{code}
was:
I think the 3 test failures in PartialIndexRebuilderIT started happening after
we switched master to HBase 1.4 as part of PHOENIX-4076.
Maybe [~lhofhansl] or [~apurtell] might have some insight
{code}
[ERROR] Failures:
[ERROR] PartialIndexRebuilderIT.testConcurrentUpsertsWithRebuild:230 Expected
equality for V1, but null!=11
[ERROR] PartialIndexRebuilderIT.testDeleteAndUpsertAfterFailure:347 Expected
equality for V2, but null!=1
[ERROR] PartialIndexRebuilderIT.testWriteWhileRebuilding:396 Expected equality
for V2, but null!=2
{code}
testDeleteAndUpsertAfterFailure and testWriteWhileRebuilding pass for me
locally just before PHOENIX-4076 was committed.
testConcurrentUpsertsWithRebuild fails with the following exception at the
commit before PHOENIX-4076 .
{code}
2018-04-27 16:14:48,049 ERROR
[RpcServer.FifoWFPBQ.default.handler=1,queue=0,port=26069]
org.apache.phoenix.coprocessor.UngroupedAggregateRegionObserver(1089):
IOException during rebuilding:
org.apache.hadoop.hbase.exceptions.TimeoutIOException: Timed out waiting for
lock for row: 80 00 00 01 80 00 00 00
at
org.apache.phoenix.hbase.index.LockManager.lockRow(LockManager.java:96)
at
org.apache.phoenix.hbase.index.Indexer.preBatchMutateWithExceptions(Indexer.java:421)
at
org.apache.phoenix.hbase.index.Indexer.preBatchMutate(Indexer.java:370)
at
org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$35.call(RegionCoprocessorHost.java:1007)
at
org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$RegionOperation.call(RegionCoprocessorHost.java:1673)
at
org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.execOperation(RegionCoprocessorHost.java:1749)
at
org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.execOperation(RegionCoprocessorHost.java:1705)
at
org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preBatchMutate(RegionCoprocessorHost.java:1003)
at
org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3190)
at
org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2976)
at
org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2918)
at
org.apache.phoenix.coprocessor.UngroupedAggregateRegionObserver.rebuildIndices(UngroupedAggregateRegionObserver.java:1074)
at
org.apache.phoenix.coprocessor.UngroupedAggregateRegionObserver.doPostScannerOpen(UngroupedAggregateRegionObserver.java:369)
at
org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.overrideDelegate(BaseScannerRegionObserver.java:245)
at
org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:293)
at
org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:2629)
at
org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:2833)
at
org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:34950)
at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2339)
at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:123)
at
org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:188)
at
org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:168)
2018-04-27 16:14:48,051 DEBUG
[RpcServer.FifoWFPBQ.default.handler=1,queue=0,port=26069]
org.apache.hadoop.hbase.ipc.CallRunner(126):
RpcServer.FifoWFPBQ.default.handler=1,queue=0,port=26069: callId: 1941 service:
ClientService methodName: Scan size: 40 connection: 127.0.0.1:14017
org.apache.hadoop.hbase.UnknownScannerException: Throwing
UnknownScannerException to reset the client scanner state for clients older
than 1.3.
at
org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:2893)
at
org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:34950)
at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2339)
at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:123)
at
org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:188)
at
org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:168)
Caused by: org.apache.hadoop.hbase.exceptions.TimeoutIOException: Timed out
waiting for lock for row: 80 00 00 01 80 00 00 00
at
org.apache.phoenix.hbase.index.LockManager.lockRow(LockManager.java:96)
at
org.apache.phoenix.hbase.index.Indexer.preBatchMutateWithExceptions(Indexer.java:421)
at
org.apache.phoenix.hbase.index.Indexer.preBatchMutate(Indexer.java:370)
at
org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$35.call(RegionCoprocessorHost.java:1007)
at
org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$RegionOperation.call(RegionCoprocessorHost.java:1673)
at
org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.execOperation(RegionCoprocessorHost.java:1749)
at
org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.execOperation(RegionCoprocessorHost.java:1705)
at
org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preBatchMutate(RegionCoprocessorHost.java:1003)
at
org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3190)
at
org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2976)
at
org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2918)
at
org.apache.phoenix.coprocessor.UngroupedAggregateRegionObserver.rebuildIndices(UngroupedAggregateRegionObserver.java:1074)
at
org.apache.phoenix.coprocessor.UngroupedAggregateRegionObserver.doPostScannerOpen(UngroupedAggregateRegionObserver.java:369)
at
org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.overrideDelegate(BaseScannerRegionObserver.java:245)
at
org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:293)
at
org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:2629)
at
org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:2833)
{code}
S
> PartialIndexRebuilderIT tests fail after switching master to HBase 1.4
> ----------------------------------------------------------------------
>
> Key: PHOENIX-4715
> URL: https://issues.apache.org/jira/browse/PHOENIX-4715
> Project: Phoenix
> Issue Type: Bug
> Reporter: Thomas D'Silva
> Priority: Major
>
> I think the 3 test failures in PartialIndexRebuilderIT started happening
> after we switched master to HBase 1.4 as part of PHOENIX-4076.
> Maybe [~lhofhansl] or [~apurtell] might have some insight
> {code:java}
> [ERROR] Failures:
> [ERROR] PartialIndexRebuilderIT.testConcurrentUpsertsWithRebuild:230 Expected
> equality for V1, but null!=11
> [ERROR] PartialIndexRebuilderIT.testDeleteAndUpsertAfterFailure:347 Expected
> equality for V2, but null!=1
> [ERROR] PartialIndexRebuilderIT.testWriteWhileRebuilding:396 Expected
> equality for V2, but null!=2
> {code}
> testDeleteAndUpsertAfterFailure and testWriteWhileRebuilding pass for me
> locally just before PHOENIX-4076 was committed.
> testConcurrentUpsertsWithRebuild fails with the following exception at the
> commit before PHOENIX-4076 .
> {code:java}
> 2018-04-27 16:14:48,049 ERROR
> [RpcServer.FifoWFPBQ.default.handler=1,queue=0,port=26069]
> org.apache.phoenix.coprocessor.UngroupedAggregateRegionObserver(1089):
> IOException during rebuilding:
> org.apache.hadoop.hbase.exceptions.TimeoutIOException: Timed out waiting for
> lock for row: 80 00 00 01 80 00 00 00
> at
> org.apache.phoenix.hbase.index.LockManager.lockRow(LockManager.java:96)
> at
> org.apache.phoenix.hbase.index.Indexer.preBatchMutateWithExceptions(Indexer.java:421)
> at
> org.apache.phoenix.hbase.index.Indexer.preBatchMutate(Indexer.java:370)
> at
> org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$35.call(RegionCoprocessorHost.java:1007)
> at
> org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$RegionOperation.call(RegionCoprocessorHost.java:1673)
> at
> org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.execOperation(RegionCoprocessorHost.java:1749)
> at
> org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.execOperation(RegionCoprocessorHost.java:1705)
> at
> org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preBatchMutate(RegionCoprocessorHost.java:1003)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3190)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2976)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2918)
> at
> org.apache.phoenix.coprocessor.UngroupedAggregateRegionObserver.rebuildIndices(UngroupedAggregateRegionObserver.java:1074)
> at
> org.apache.phoenix.coprocessor.UngroupedAggregateRegionObserver.doPostScannerOpen(UngroupedAggregateRegionObserver.java:369)
> at
> org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.overrideDelegate(BaseScannerRegionObserver.java:245)
> at
> org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:293)
> at
> org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:2629)
> at
> org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:2833)
> at
> org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:34950)
> at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2339)
> at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:123)
> at
> org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:188)
> at
> org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:168)
> 2018-04-27 16:14:48,051 DEBUG
> [RpcServer.FifoWFPBQ.default.handler=1,queue=0,port=26069]
> org.apache.hadoop.hbase.ipc.CallRunner(126):
> RpcServer.FifoWFPBQ.default.handler=1,queue=0,port=26069: callId: 1941
> service: ClientService methodName: Scan size: 40 connection: 127.0.0.1:14017
> org.apache.hadoop.hbase.UnknownScannerException: Throwing
> UnknownScannerException to reset the client scanner state for clients older
> than 1.3.
> at
> org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:2893)
> at
> org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:34950)
> at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2339)
> at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:123)
> at
> org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:188)
> at
> org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:168)
> Caused by: org.apache.hadoop.hbase.exceptions.TimeoutIOException: Timed out
> waiting for lock for row: 80 00 00 01 80 00 00 00
> at
> org.apache.phoenix.hbase.index.LockManager.lockRow(LockManager.java:96)
> at
> org.apache.phoenix.hbase.index.Indexer.preBatchMutateWithExceptions(Indexer.java:421)
> at
> org.apache.phoenix.hbase.index.Indexer.preBatchMutate(Indexer.java:370)
> at
> org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$35.call(RegionCoprocessorHost.java:1007)
> at
> org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$RegionOperation.call(RegionCoprocessorHost.java:1673)
> at
> org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.execOperation(RegionCoprocessorHost.java:1749)
> at
> org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.execOperation(RegionCoprocessorHost.java:1705)
> at
> org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preBatchMutate(RegionCoprocessorHost.java:1003)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3190)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2976)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2918)
> at
> org.apache.phoenix.coprocessor.UngroupedAggregateRegionObserver.rebuildIndices(UngroupedAggregateRegionObserver.java:1074)
> at
> org.apache.phoenix.coprocessor.UngroupedAggregateRegionObserver.doPostScannerOpen(UngroupedAggregateRegionObserver.java:369)
> at
> org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.overrideDelegate(BaseScannerRegionObserver.java:245)
> at
> org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:293)
> at
> org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:2629)
> at
> org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:2833)
> {code}
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
