gaofeng created HBASE-27465: ------------------------------- Summary: The regionserver service process is in, but the hmaster page shows dead Key: HBASE-27465 URL: https://issues.apache.org/jira/browse/HBASE-27465 Project: HBase Issue Type: Bug Components: scan Affects Versions: 2.3.5 Reporter: gaofeng
hmaster log {{{}2022{}}}{{{}-{}}}{{{}11{}}}{{{}-{}}}{{{}03{}}} {{{}02{}}}{{{}:{}}}{{{}06{}}}{{{}:{}}}{{{}14{}}}{{{},{}}}{{{}002{}}} {{{}INFO [RegionServerTracker-{}}}{{{}0{}}}{{{}] master.RegionServerTracker: RegionServer ephemeral node deleted, processing expiration [dx-pipe-sata674{}}} {{{}-pm,{}}}{{{}16020{}}}{{{},{}}}{{{}1667070800282{}}}{{{}]{}}} {{{}2022{}}}{{{}-{}}}{{{}11{}}}{{{}-{}}}{{{}03{}}} {{{}02{}}}{{{}:{}}}{{{}06{}}}{{{}:{}}}{{{}14{}}}{{{},{}}}{{{}002{}}} {{{}INFO [RegionServerTracker-{}}}{{{}0{}}}{{{}] master.ServerManager: Processing expiration of dx-pipe-sata674-pm,{}}}{{{}16020{}}}{{{},{}}}{{{}1667070800282{}}} {{on dx-pipe-pt263-}} {{{}pm,{}}}{{{}16000{}}}{{{},{}}}{{{}1632381868857{}}} {{}} regionserver log {{{}2022{}}}{{{}-{}}}{{{}11{}}}{{{}-{}}}{{{}03{}}} {{{}02{}}}{{{}:{}}}{{{}05{}}}{{{}:{}}}{{{}37{}}}{{{},{}}}{{{}114{}}} {{{}WARN [RpcServer.{}}}{{{}default{}}}{{{}.FPBQ.Fifo.handler={}}}{{{}22{}}}{{{},queue={}}}{{{}2{}}}{{{},port={}}}{{{}16020{}}}{{{}] hfile.BlockCacheUtil: Caching an already cached block: {}}}{{683273027345}} {{{}48448d8384bea82dd541_SeqId_6__346590844. This is harmless and can happen in rare cases (see HBASE-{}}}{{{}8547{}}}{{{}){}}} {{{}2022{}}}{{{}-{}}}{{{}11{}}}{{{}-{}}}{{{}03{}}} {{{}02{}}}{{{}:{}}}{{{}05{}}}{{{}:{}}}{{{}37{}}}{{{},{}}}{{{}114{}}} {{{}WARN [RpcServer.{}}}{{{}default{}}}{{{}.FPBQ.Fifo.handler={}}}{{{}43{}}}{{{},queue={}}}{{{}3{}}}{{{},port={}}}{{{}16020{}}}{{{}] hfile.BlockCacheUtil: Cached block contents differ by nextBlockOnDi{}}} {{skSize, the }}{{new}} {{block has nextBlockOnDiskSize set. Caching }}{{new}} {{block.}} {{{}2022{}}}{{{}-{}}}{{{}11{}}}{{{}-{}}}{{{}03{}}} {{{}02{}}}{{{}:{}}}{{{}05{}}}{{{}:{}}}{{{}37{}}}{{{},{}}}{{{}115{}}} {{{}ERROR [RpcServer.{}}}{{{}default{}}}{{{}.FPBQ.Fifo.handler={}}}{{{}22{}}}{{{},queue={}}}{{{}2{}}}{{{},port={}}}{{{}16020{}}}{{{}] ipc.RpcServer: Unexpected throwable object{}}} {{{}java.lang.IllegalArgumentException: offset ({}}}{{{}0{}}}{{{}) + length ({}}}{{{}2{}}}{{{}) exceed the capacity of the array: -{}}}{{{}1207959040{}}} {{ }}{{{}at org.apache.hadoop.hbase.util.Bytes.explainWrongLengthOrOffset(Bytes.java:{}}}{{{}832{}}}{{{}){}}} {{ }}{{{}at org.apache.hadoop.hbase.util.Bytes.toShort(Bytes.java:{}}}{{{}1125{}}}{{{}){}}} {{ }}{{{}at org.apache.hadoop.hbase.util.Bytes.toShort(Bytes.java:{}}}{{{}1111{}}}{{{}){}}} {{ }}{{{}at org.apache.hadoop.hbase.io.encoding.DiffKeyDeltaEncoder$DiffSeekerStateBufferedEncodedSeeker.decode(DiffKeyDeltaEncoder.java:{}}}{{{}439{}}}{{{}){}}} {{ }}{{{}at org.apache.hadoop.hbase.io.encoding.DiffKeyDeltaEncoder$DiffSeekerStateBufferedEncodedSeeker.decodeNext(DiffKeyDeltaEncoder.java:{}}}{{{}527{}}}{{{}){}}} {{ }}{{{}at org.apache.hadoop.hbase.io.encoding.BufferedDataBlockEncoder$BufferedEncodedSeeker.next(BufferedDataBlockEncoder.java:{}}}{{{}806{}}}{{{}){}}} {{ }}{{{}at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:{}}}{{{}1512{}}}{{{}){}}} {{ }}{{{}at org.apache.hadoop.hbase.regionserver.StoreFileScanner.next(StoreFileScanner.java:{}}}{{{}196{}}}{{{}){}}} {{ }}{{{}at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:{}}}{{{}124{}}}{{{}){}}} {{ }}{{{}at org.apache.hadoop.hbase.regionserver.StoreScanner.trySkipToNextColumn(StoreScanner.java:{}}}{{{}917{}}}{{{}){}}} {{ }}{{{}at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:{}}}{{{}826{}}}{{{}){}}} {{ }}{{{}at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:{}}}{{{}738{}}}{{{}){}}} {{ }}{{{}at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:{}}}{{{}157{}}}{{{}){}}} {{ }}{{{}at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:{}}}{{{}6794{}}}{{{}){}}} {{ }}{{{}at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:{}}}{{{}6958{}}}{{{}){}}} {{ }}{{{}at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:{}}}{{{}6728{}}}{{{}){}}} {{ }}{{{}at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:{}}}{{{}3323{}}}{{{}){}}} {{ }}{{{}at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:{}}}{{{}3574{}}}{{{}){}}} {{ }}{{{}at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService${}}}{{{}2{}}}{{{}.callBlockingMethod(ClientProtos.java:{}}}{{{}44858{}}}{{{}){}}} {{ }}{{{}at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:{}}}{{{}393{}}}{{{}){}}} {{ }}{{{}at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:{}}}{{{}133{}}}{{{}){}}} {{ }}{{{}at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:{}}}{{{}338{}}}{{{}){}}} {{ }}{{{}at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:{}}}{{{}318{}}}{{{}){}}} {{{}2022{}}}{{{}-{}}}{{{}11{}}}{{{}-{}}}{{{}03{}}} {{{}02{}}}{{{}:{}}}{{{}05{}}}{{{}:{}}}{{{}37{}}}{{{},{}}}{{{}116{}}} {{{}WARN [RpcServer.{}}}{{{}default{}}}{{{}.FPBQ.Fifo.handler={}}}{{{}13{}}}{{{},queue={}}}{{{}3{}}}{{{},port={}}}{{{}16020{}}}{{{}] hfile.BlockCacheUtil: Caching an already cached block: 297c0c67dd7840b5829c4dae5342664b_SeqId_4__346614261. This is harmless and can happen in rare cases (see HBASE-{}}}{{{}8547{}}}{{{}){}}} {{{}2022{}}}{{{}-{}}}{{{}11{}}}{{{}-{}}}{{{}03{}}} {{{}02{}}}{{{}:{}}}{{{}05{}}}{{{}:{}}}{{{}37{}}}{{{},{}}}{{{}117{}}} {{{}WARN [RpcServer.{}}}{{{}default{}}}{{{}.FPBQ.Fifo.handler={}}}{{{}13{}}}{{{},queue={}}}{{{}3{}}}{{{},port={}}}{{{}16020{}}}{{{}] hfile.BlockCacheUtil: Caching an already cached block: 68327302734548448d8384bea82dd541_SeqId_6__346656355. This is harmless and can happen in rare cases (see HBASE-{}}}{{{}8547{}}}{{{}){}}} -- This message was sent by Atlassian Jira (v8.20.10#820010)