[
https://issues.apache.org/jira/browse/HDDS-10632?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Wei-Chiu Chuang resolved HDDS-10632.
------------------------------------
Fix Version/s: HDDS-7593
Resolution: Fixed
> [Hbase Ozone] HMaster aborted with "IOException: Inconsistent read"
> -------------------------------------------------------------------
>
> Key: HDDS-10632
> URL: https://issues.apache.org/jira/browse/HDDS-10632
> Project: Apache Ozone
> Issue Type: Bug
> Components: SCM
> Reporter: Pratyush Bhatt
> Assignee: Ashish Kumar
> Priority: Major
> Labels: pull-request-available
> Fix For: HDDS-7593
>
>
> Both the HMasters are down, the HMaster fails with:
> {code:java}
> 2024-04-01 13:15:51,517 ERROR org.apache.hadoop.hbase.master.HMaster: Failed
> to become active master
> java.io.IOException: Inconsistent read for blockID=conID: 8366 locID:
> 113750153625964072 bcsId: 0 length=268435456 position=83 numBytesToRead=1
> numBytesRead=-1
> at
> org.apache.hadoop.ozone.client.io.KeyInputStream.checkPartBytesRead(KeyInputStream.java:191)
> at
> org.apache.hadoop.hdds.scm.storage.MultipartInputStream.readWithStrategy(MultipartInputStream.java:97)
> at
> org.apache.hadoop.hdds.scm.storage.ExtendedInputStream.read(ExtendedInputStream.java:56)
> at
> org.apache.hadoop.hdds.scm.storage.ExtendedInputStream.read(ExtendedInputStream.java:43)
> at
> org.apache.hadoop.fs.ozone.OzoneFSInputStream.read(OzoneFSInputStream.java:55)
> at java.io.FilterInputStream.read(FilterInputStream.java:83)
> at
> org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.parseDelimitedFrom(ProtobufUtil.java:3576)
> at
> org.apache.hadoop.hbase.regionserver.wal.ProtobufLogReader.readNext(ProtobufLogReader.java:348)
> at
> org.apache.hadoop.hbase.regionserver.wal.ReaderBase.next(ReaderBase.java:95)
> at
> org.apache.hadoop.hbase.regionserver.wal.ReaderBase.next(ReaderBase.java:83)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.replayRecoveredEdits(HRegion.java:5298)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.replayRecoveredEditsIfAny(HRegion.java:5182)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.initializeRegionInternals(HRegion.java:998)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.initialize(HRegion.java:939)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7903)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.openHRegionFromTableDir(HRegion.java:7860)
> at
> org.apache.hadoop.hbase.master.region.MasterRegion.open(MasterRegion.java:307)
> at
> org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:424)
> at
> org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:122)
> at
> org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:848)
> at
> org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2216)
> at
> org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:528)
> at java.lang.Thread.run(Thread.java:748)
> 2024-04-01 13:15:51,517 ERROR org.apache.hadoop.hbase.master.HMaster: *****
> ABORTING master vc0121.halxg.cloudera.com,22001,1711989581483: Unhandled
> exception. Starting shutdown. *****
> java.io.IOException: Inconsistent read for blockID=conID: 8366 locID:
> 113750153625964072 bcsId: 0 length=268435456 position=83 numBytesToRead=1
> numBytesRead=-1
> at
> org.apache.hadoop.ozone.client.io.KeyInputStream.checkPartBytesRead(KeyInputStream.java:191)
> at
> org.apache.hadoop.hdds.scm.storage.MultipartInputStream.readWithStrategy(MultipartInputStream.java:97)
> at
> org.apache.hadoop.hdds.scm.storage.ExtendedInputStream.read(ExtendedInputStream.java:56)
> at
> org.apache.hadoop.hdds.scm.storage.ExtendedInputStream.read(ExtendedInputStream.java:43)
> at
> org.apache.hadoop.fs.ozone.OzoneFSInputStream.read(OzoneFSInputStream.java:55)
> at java.io.FilterInputStream.read(FilterInputStream.java:83)
> at
> org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.parseDelimitedFrom(ProtobufUtil.java:3576)
> at
> org.apache.hadoop.hbase.regionserver.wal.ProtobufLogReader.readNext(ProtobufLogReader.java:348)
> at
> org.apache.hadoop.hbase.regionserver.wal.ReaderBase.next(ReaderBase.java:95)
> at
> org.apache.hadoop.hbase.regionserver.wal.ReaderBase.next(ReaderBase.java:83)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.replayRecoveredEdits(HRegion.java:5298)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.replayRecoveredEditsIfAny(HRegion.java:5182)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.initializeRegionInternals(HRegion.java:998)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.initialize(HRegion.java:939)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7903)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.openHRegionFromTableDir(HRegion.java:7860)
> at
> org.apache.hadoop.hbase.master.region.MasterRegion.open(MasterRegion.java:307)
> at
> org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:424)
> at
> org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:122)
> at
> org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:848)
> at
> org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2216)
> at
> org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:528)
> at java.lang.Thread.run(Thread.java:748)
> 2024-04-01 13:15:51,517 INFO
> org.apache.hadoop.hbase.regionserver.HRegionServer: ***** STOPPING region
> server 'vc0121.xyz,22001,1711989581483' ***** {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@ozone.apache.org
For additional commands, e-mail: issues-h...@ozone.apache.org
