[
https://issues.apache.org/jira/browse/HBASE-27465?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
gaofeng updated HBASE-27465:
----------------------------
Description:
hmaster log info
{{{}2022{}}}{{{}-{}}}{{{}11{}}}{{{}-{}}}{{{}03{}}}
{{{}02{}}}{{{}:{}}}{{{}06{}}}{{{}:{}}}{{{}14{}}}{{{},{}}}{{{}002{}}} {{{}INFO
[RegionServerTracker-{}}}{{{}0{}}}{{{}] master.RegionServerTracker:
RegionServer ephemeral node deleted, processing expiration [dx-pipe-sata674{}}}
{{{}-pm,{}}}{{{}16020{}}}{{{},{}}}{{{}1667070800282{}}}{{{}]{}}}
{{{}2022{}}}{{{}-{}}}{{{}11{}}}{{{}-{}}}{{{}03{}}}
{{{}02{}}}{{{}:{}}}{{{}06{}}}{{{}:{}}}{{{}14{}}}{{{},{}}}{{{}002{}}} {{{}INFO
[RegionServerTracker-{}}}{{{}0{}}}{{{}] master.ServerManager: Processing
expiration of
dx-pipe-sata674-pm,{}}}{{{}16020{}}}{{{},{}}}{{{}1667070800282{}}} {{on
dx-pipe-pt263-}}
{{{}pm,{}}}{{{}16000{}}}{{{},{}}}{{{}1632381868857{}}}
{{{}2022{}}}{{{}-{}}}{{{}11{}}}{{{}-{}}}{{{}03{}}}
{{{}02{}}}{{{}:{}}}{{{}06{}}}{{{}:{}}}{{{}14{}}}{{{},{}}}{{{}006{}}} {{{}INFO
[RegionServerTracker-{}}}{{{}0{}}}{{{}] assignment.AssignmentManager: Scheduled
ServerCrashProcedure pid={}}}{{{}4251436{}}} {{for}}
{{{}dx-pipe-sata674-pm,{}}}{{{}16020{}}}{{{},{}}}{{{}1{}}}
{{667070800282}} {{{}(carryingMeta={}}}{{{}false{}}}{{{})
dx-pipe-sata674-pm,{}}}{{{}16020{}}}{{{},{}}}{{{}1667070800282{}}}{{{}/CRASHED/regionCount={}}}{{{}522{}}}{{{}/lock=java.util.concurrent.locks.ReentrantReadWriteLock{}}}{{{}@938cb{}}}
{{{}01{}}}{{{}[Write locks = {}}}{{{}1{}}}{{{}, Read locks =
{}}}{{{}0{}}}{{{}], oldState=ONLINE.{}}}
{{{}2022{}}}{{{}-{}}}{{{}11{}}}{{{}-{}}}{{{}03{}}}
{{{}02{}}}{{{}:{}}}{{{}06{}}}{{{}:{}}}{{{}14{}}}{{{},{}}}{{{}006{}}} {{{}INFO
[org.apache.hadoop.hbase.rsgroup.RSGroupInfoManagerImpl$ServerEventsListenerThread-dx-pipe-pt263-pm,{}}}{{{}16000{}}}{{{},{}}}{{{}1632381868857{}}}{{{}]
rsgrou{}}}
{{p.RSGroupInfoManagerImpl$ServerEventsListenerThread: Updating }}{{default}}
{{servers.}}
{{{}2022{}}}{{{}-{}}}{{{}11{}}}{{{}-{}}}{{{}03{}}}
{{{}02{}}}{{{}:{}}}{{{}06{}}}{{{}:{}}}{{{}14{}}}{{{},{}}}{{{}007{}}} {{{}INFO
[PEWorker-{}}}{{{}9{}}}{{{}] procedure.ServerCrashProcedure: Start
pid={}}}{{{}4251436{}}}{{{}, state=RUNNABLE:SERVER_CRASH_START,
locked={}}}{{{}true{}}}{{{}; ServerCrashProc{}}}
{{{}edure
dx-pipe-sata674-pm,{}}}{{{}16020{}}}{{{},{}}}{{{}1667070800282{}}}{{{},
splitWal={}}}{{{}true{}}}{{{}, meta={}}}{{{}false{}}}
{{{}2022{}}}{{{}-{}}}{{{}11{}}}{{{}-{}}}{{{}03{}}}
{{{}02{}}}{{{}:{}}}{{{}06{}}}{{{}:{}}}{{{}14{}}}{{{},{}}}{{{}007{}}} {{{}INFO
[PEWorker-{}}}{{{}9{}}}{{{}] procedure.ServerCrashProcedure:
dx-pipe-sata674-pm,{}}}{{{}16020{}}}{{{},{}}}{{{}1667070800282{}}} {{had
}}{{522}} {{regions}}
{{{}2022{}}}{{{}-{}}}{{{}11{}}}{{{}-{}}}{{{}03{}}}
{{{}02{}}}{{{}:{}}}{{{}06{}}}{{{}:{}}}{{{}14{}}}{{{},{}}}{{{}016{}}} {{{}INFO
[PEWorker-{}}}{{{}9{}}}{{{}] master.SplitLogManager: dead splitlog workers
[dx-pipe-sata674-pm,{}}}{{{}16020{}}}{{{},{}}}{{{}1667070800282{}}}{{{}]{}}}
{{{}2022{}}}{{{}-{}}}{{{}11{}}}{{{}-{}}}{{{}03{}}}
{{{}02{}}}{{{}:{}}}{{{}06{}}}{{{}:{}}}{{{}14{}}}{{{},{}}}{{{}016{}}} {{{}INFO
[PEWorker-{}}}{{{}9{}}}{{{}] master.SplitLogManager: Started splitting
{}}}{{3}} {{{}logs in
[hdfs:{}}}{{{}//bigdata-hbase-hdd/hbase/WALs/dx-pipe-sata674-pm,16020,{}}}
{{{}1667070800282{}}}{{{}-splitting] {}}}{{for}}
{{{}[dx-pipe-sata674-pm,{}}}{{{}16020{}}}{{{},{}}}{{{}1667070800282{}}}{{{}]{}}}
was:
hmaster log
{{{}2022{}}}{{{}-{}}}{{{}11{}}}{{{}-{}}}{{{}03{}}}
{{{}02{}}}{{{}:{}}}{{{}06{}}}{{{}:{}}}{{{}14{}}}{{{},{}}}{{{}002{}}} {{{}INFO
[RegionServerTracker-{}}}{{{}0{}}}{{{}] master.RegionServerTracker:
RegionServer ephemeral node deleted, processing expiration [dx-pipe-sata674{}}}
{{{}-pm,{}}}{{{}16020{}}}{{{},{}}}{{{}1667070800282{}}}{{{}]{}}}
{{{}2022{}}}{{{}-{}}}{{{}11{}}}{{{}-{}}}{{{}03{}}}
{{{}02{}}}{{{}:{}}}{{{}06{}}}{{{}:{}}}{{{}14{}}}{{{},{}}}{{{}002{}}} {{{}INFO
[RegionServerTracker-{}}}{{{}0{}}}{{{}] master.ServerManager: Processing
expiration of
dx-pipe-sata674-pm,{}}}{{{}16020{}}}{{{},{}}}{{{}1667070800282{}}} {{on
dx-pipe-pt263-}}
{{{}pm,{}}}{{{}16000{}}}{{{},{}}}{{{}1632381868857{}}}
{{}}
regionserver log
{{{}2022{}}}{{{}-{}}}{{{}11{}}}{{{}-{}}}{{{}03{}}}
{{{}02{}}}{{{}:{}}}{{{}05{}}}{{{}:{}}}{{{}37{}}}{{{},{}}}{{{}114{}}} {{{}WARN
[RpcServer.{}}}{{{}default{}}}{{{}.FPBQ.Fifo.handler={}}}{{{}22{}}}{{{},queue={}}}{{{}2{}}}{{{},port={}}}{{{}16020{}}}{{{}]
hfile.BlockCacheUtil: Caching an already cached block: {}}}{{683273027345}}
{{{}48448d8384bea82dd541_SeqId_6__346590844. This is harmless and can happen in
rare cases (see HBASE-{}}}{{{}8547{}}}{{{}){}}}
{{{}2022{}}}{{{}-{}}}{{{}11{}}}{{{}-{}}}{{{}03{}}}
{{{}02{}}}{{{}:{}}}{{{}05{}}}{{{}:{}}}{{{}37{}}}{{{},{}}}{{{}114{}}} {{{}WARN
[RpcServer.{}}}{{{}default{}}}{{{}.FPBQ.Fifo.handler={}}}{{{}43{}}}{{{},queue={}}}{{{}3{}}}{{{},port={}}}{{{}16020{}}}{{{}]
hfile.BlockCacheUtil: Cached block contents differ by nextBlockOnDi{}}}
{{skSize, the }}{{new}} {{block has nextBlockOnDiskSize set. Caching }}{{new}}
{{block.}}
{{{}2022{}}}{{{}-{}}}{{{}11{}}}{{{}-{}}}{{{}03{}}}
{{{}02{}}}{{{}:{}}}{{{}05{}}}{{{}:{}}}{{{}37{}}}{{{},{}}}{{{}115{}}} {{{}ERROR
[RpcServer.{}}}{{{}default{}}}{{{}.FPBQ.Fifo.handler={}}}{{{}22{}}}{{{},queue={}}}{{{}2{}}}{{{},port={}}}{{{}16020{}}}{{{}]
ipc.RpcServer: Unexpected throwable object{}}}
{{{}java.lang.IllegalArgumentException: offset ({}}}{{{}0{}}}{{{}) + length
({}}}{{{}2{}}}{{{}) exceed the capacity of the array: -{}}}{{{}1207959040{}}}
{{ }}{{{}at
org.apache.hadoop.hbase.util.Bytes.explainWrongLengthOrOffset(Bytes.java:{}}}{{{}832{}}}{{{}){}}}
{{ }}{{{}at
org.apache.hadoop.hbase.util.Bytes.toShort(Bytes.java:{}}}{{{}1125{}}}{{{}){}}}
{{ }}{{{}at
org.apache.hadoop.hbase.util.Bytes.toShort(Bytes.java:{}}}{{{}1111{}}}{{{}){}}}
{{ }}{{{}at
org.apache.hadoop.hbase.io.encoding.DiffKeyDeltaEncoder$DiffSeekerStateBufferedEncodedSeeker.decode(DiffKeyDeltaEncoder.java:{}}}{{{}439{}}}{{{}){}}}
{{ }}{{{}at
org.apache.hadoop.hbase.io.encoding.DiffKeyDeltaEncoder$DiffSeekerStateBufferedEncodedSeeker.decodeNext(DiffKeyDeltaEncoder.java:{}}}{{{}527{}}}{{{}){}}}
{{ }}{{{}at
org.apache.hadoop.hbase.io.encoding.BufferedDataBlockEncoder$BufferedEncodedSeeker.next(BufferedDataBlockEncoder.java:{}}}{{{}806{}}}{{{}){}}}
{{ }}{{{}at
org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:{}}}{{{}1512{}}}{{{}){}}}
{{ }}{{{}at
org.apache.hadoop.hbase.regionserver.StoreFileScanner.next(StoreFileScanner.java:{}}}{{{}196{}}}{{{}){}}}
{{ }}{{{}at
org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:{}}}{{{}124{}}}{{{}){}}}
{{ }}{{{}at
org.apache.hadoop.hbase.regionserver.StoreScanner.trySkipToNextColumn(StoreScanner.java:{}}}{{{}917{}}}{{{}){}}}
{{ }}{{{}at
org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:{}}}{{{}826{}}}{{{}){}}}
{{ }}{{{}at
org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:{}}}{{{}738{}}}{{{}){}}}
{{ }}{{{}at
org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:{}}}{{{}157{}}}{{{}){}}}
{{ }}{{{}at
org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:{}}}{{{}6794{}}}{{{}){}}}
{{ }}{{{}at
org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:{}}}{{{}6958{}}}{{{}){}}}
{{ }}{{{}at
org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:{}}}{{{}6728{}}}{{{}){}}}
{{ }}{{{}at
org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:{}}}{{{}3323{}}}{{{}){}}}
{{ }}{{{}at
org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:{}}}{{{}3574{}}}{{{}){}}}
{{ }}{{{}at
org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService${}}}{{{}2{}}}{{{}.callBlockingMethod(ClientProtos.java:{}}}{{{}44858{}}}{{{}){}}}
{{ }}{{{}at
org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:{}}}{{{}393{}}}{{{}){}}}
{{ }}{{{}at
org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:{}}}{{{}133{}}}{{{}){}}}
{{ }}{{{}at
org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:{}}}{{{}338{}}}{{{}){}}}
{{ }}{{{}at
org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:{}}}{{{}318{}}}{{{}){}}}
{{{}2022{}}}{{{}-{}}}{{{}11{}}}{{{}-{}}}{{{}03{}}}
{{{}02{}}}{{{}:{}}}{{{}05{}}}{{{}:{}}}{{{}37{}}}{{{},{}}}{{{}116{}}} {{{}WARN
[RpcServer.{}}}{{{}default{}}}{{{}.FPBQ.Fifo.handler={}}}{{{}13{}}}{{{},queue={}}}{{{}3{}}}{{{},port={}}}{{{}16020{}}}{{{}]
hfile.BlockCacheUtil: Caching an already cached block:
297c0c67dd7840b5829c4dae5342664b_SeqId_4__346614261. This is harmless and can
happen in rare cases (see HBASE-{}}}{{{}8547{}}}{{{}){}}}
{{{}2022{}}}{{{}-{}}}{{{}11{}}}{{{}-{}}}{{{}03{}}}
{{{}02{}}}{{{}:{}}}{{{}05{}}}{{{}:{}}}{{{}37{}}}{{{},{}}}{{{}117{}}} {{{}WARN
[RpcServer.{}}}{{{}default{}}}{{{}.FPBQ.Fifo.handler={}}}{{{}13{}}}{{{},queue={}}}{{{}3{}}}{{{},port={}}}{{{}16020{}}}{{{}]
hfile.BlockCacheUtil: Caching an already cached block:
68327302734548448d8384bea82dd541_SeqId_6__346656355. This is harmless and can
happen in rare cases (see HBASE-{}}}{{{}8547{}}}{{{}){}}}
> The regionserver service process is in, but the hmaster page shows dead
> -----------------------------------------------------------------------
>
> Key: HBASE-27465
> URL: https://issues.apache.org/jira/browse/HBASE-27465
> Project: HBase
> Issue Type: Bug
> Components: scan
> Affects Versions: 2.3.5
> Reporter: gaofeng
> Priority: Major
>
> hmaster log info
> {{{}2022{}}}{{{}-{}}}{{{}11{}}}{{{}-{}}}{{{}03{}}}
> {{{}02{}}}{{{}:{}}}{{{}06{}}}{{{}:{}}}{{{}14{}}}{{{},{}}}{{{}002{}}} {{{}INFO
> [RegionServerTracker-{}}}{{{}0{}}}{{{}] master.RegionServerTracker:
> RegionServer ephemeral node deleted, processing expiration
> [dx-pipe-sata674{}}}
> {{{}-pm,{}}}{{{}16020{}}}{{{},{}}}{{{}1667070800282{}}}{{{}]{}}}
> {{{}2022{}}}{{{}-{}}}{{{}11{}}}{{{}-{}}}{{{}03{}}}
> {{{}02{}}}{{{}:{}}}{{{}06{}}}{{{}:{}}}{{{}14{}}}{{{},{}}}{{{}002{}}} {{{}INFO
> [RegionServerTracker-{}}}{{{}0{}}}{{{}] master.ServerManager: Processing
> expiration of
> dx-pipe-sata674-pm,{}}}{{{}16020{}}}{{{},{}}}{{{}1667070800282{}}} {{on
> dx-pipe-pt263-}}
> {{{}pm,{}}}{{{}16000{}}}{{{},{}}}{{{}1632381868857{}}}
> {{{}2022{}}}{{{}-{}}}{{{}11{}}}{{{}-{}}}{{{}03{}}}
> {{{}02{}}}{{{}:{}}}{{{}06{}}}{{{}:{}}}{{{}14{}}}{{{},{}}}{{{}006{}}} {{{}INFO
> [RegionServerTracker-{}}}{{{}0{}}}{{{}] assignment.AssignmentManager:
> Scheduled ServerCrashProcedure pid={}}}{{{}4251436{}}} {{for}}
> {{{}dx-pipe-sata674-pm,{}}}{{{}16020{}}}{{{},{}}}{{{}1{}}}
> {{667070800282}} {{{}(carryingMeta={}}}{{{}false{}}}{{{})
> dx-pipe-sata674-pm,{}}}{{{}16020{}}}{{{},{}}}{{{}1667070800282{}}}{{{}/CRASHED/regionCount={}}}{{{}522{}}}{{{}/lock=java.util.concurrent.locks.ReentrantReadWriteLock{}}}{{{}@938cb{}}}
> {{{}01{}}}{{{}[Write locks = {}}}{{{}1{}}}{{{}, Read locks =
> {}}}{{{}0{}}}{{{}], oldState=ONLINE.{}}}
> {{{}2022{}}}{{{}-{}}}{{{}11{}}}{{{}-{}}}{{{}03{}}}
> {{{}02{}}}{{{}:{}}}{{{}06{}}}{{{}:{}}}{{{}14{}}}{{{},{}}}{{{}006{}}} {{{}INFO
>
> [org.apache.hadoop.hbase.rsgroup.RSGroupInfoManagerImpl$ServerEventsListenerThread-dx-pipe-pt263-pm,{}}}{{{}16000{}}}{{{},{}}}{{{}1632381868857{}}}{{{}]
> rsgrou{}}}
> {{p.RSGroupInfoManagerImpl$ServerEventsListenerThread: Updating }}{{default}}
> {{servers.}}
> {{{}2022{}}}{{{}-{}}}{{{}11{}}}{{{}-{}}}{{{}03{}}}
> {{{}02{}}}{{{}:{}}}{{{}06{}}}{{{}:{}}}{{{}14{}}}{{{},{}}}{{{}007{}}} {{{}INFO
> [PEWorker-{}}}{{{}9{}}}{{{}] procedure.ServerCrashProcedure: Start
> pid={}}}{{{}4251436{}}}{{{}, state=RUNNABLE:SERVER_CRASH_START,
> locked={}}}{{{}true{}}}{{{}; ServerCrashProc{}}}
> {{{}edure
> dx-pipe-sata674-pm,{}}}{{{}16020{}}}{{{},{}}}{{{}1667070800282{}}}{{{},
> splitWal={}}}{{{}true{}}}{{{}, meta={}}}{{{}false{}}}
> {{{}2022{}}}{{{}-{}}}{{{}11{}}}{{{}-{}}}{{{}03{}}}
> {{{}02{}}}{{{}:{}}}{{{}06{}}}{{{}:{}}}{{{}14{}}}{{{},{}}}{{{}007{}}} {{{}INFO
> [PEWorker-{}}}{{{}9{}}}{{{}] procedure.ServerCrashProcedure:
> dx-pipe-sata674-pm,{}}}{{{}16020{}}}{{{},{}}}{{{}1667070800282{}}} {{had
> }}{{522}} {{regions}}
>
> {{{}2022{}}}{{{}-{}}}{{{}11{}}}{{{}-{}}}{{{}03{}}}
> {{{}02{}}}{{{}:{}}}{{{}06{}}}{{{}:{}}}{{{}14{}}}{{{},{}}}{{{}016{}}} {{{}INFO
> [PEWorker-{}}}{{{}9{}}}{{{}] master.SplitLogManager: dead splitlog workers
> [dx-pipe-sata674-pm,{}}}{{{}16020{}}}{{{},{}}}{{{}1667070800282{}}}{{{}]{}}}
> {{{}2022{}}}{{{}-{}}}{{{}11{}}}{{{}-{}}}{{{}03{}}}
> {{{}02{}}}{{{}:{}}}{{{}06{}}}{{{}:{}}}{{{}14{}}}{{{},{}}}{{{}016{}}} {{{}INFO
> [PEWorker-{}}}{{{}9{}}}{{{}] master.SplitLogManager: Started splitting
> {}}}{{3}} {{{}logs in
> [hdfs:{}}}{{{}//bigdata-hbase-hdd/hbase/WALs/dx-pipe-sata674-pm,16020,{}}}
> {{{}1667070800282{}}}{{{}-splitting] {}}}{{for}}
> {{{}[dx-pipe-sata674-pm,{}}}{{{}16020{}}}{{{},{}}}{{{}1667070800282{}}}{{{}]{}}}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
