[ https://issues.apache.org/jira/browse/HBASE-22433?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16841937#comment-16841937 ]
binlijin edited comment on HBASE-22433 at 5/17/19 6:20 AM: ----------------------------------------------------------- {code} 2019-05-17 12:42:53,319 WARN [RS_OPEN_REGION-regionserver/hb-mbasedata-3:16020-0] regionserver.HRegion: Failed initialize of region=mk_usr_trce,6221a8e3c6a-34f4-4705-9d30-1dc3f235cabc155793,1558068172237.819b2a84f5de78ed3d6ec10061538d6c., starting to roll back memstore java.io.IOException: java.io.IOException: java.lang.NullPointerException at org.apache.hadoop.hbase.regionserver.HRegion.initializeStores(HRegion.java:1106) at org.apache.hadoop.hbase.regionserver.HRegion.initializeStores(HRegion.java:1048) at org.apache.hadoop.hbase.regionserver.HRegion.initializeRegionInternals(HRegion.java:959) at org.apache.hadoop.hbase.regionserver.HRegion.initialize(HRegion.java:915) at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7293) at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7252) at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7224) at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7182) at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7133) at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:133) at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: java.io.IOException: java.lang.NullPointerException at org.apache.hadoop.hbase.regionserver.HStore.openStoreFiles(HStore.java:593) at org.apache.hadoop.hbase.regionserver.HStore.loadStoreFiles(HStore.java:557) at org.apache.hadoop.hbase.regionserver.HStore.<init>(HStore.java:303) at org.apache.hadoop.hbase.regionserver.HRegion.instantiateHStore(HRegion.java:5778) at org.apache.hadoop.hbase.regionserver.HRegion$1.call(HRegion.java:1070) at org.apache.hadoop.hbase.regionserver.HRegion$1.call(HRegion.java:1067) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) ... 3 more Caused by: java.lang.NullPointerException at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.returnBlock(BucketCache.java:1665) at org.apache.hadoop.hbase.io.hfile.BlockCacheUtil.shouldReplaceExistingCacheBlock(BlockCacheUtil.java:250) at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.cacheBlockWithWait(BucketCache.java:426) at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.cacheBlock(BucketCache.java:412) at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.cacheBlock(CombinedBlockCache.java:67) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.lambda$readBlock$2(HFileReaderImpl.java:1501) at java.util.Optional.ifPresent(Optional.java:159) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1499) at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$CellBasedKeyBlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:340) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.seekTo(HFileReaderImpl.java:856) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.seekTo(HFileReaderImpl.java:806) at org.apache.hadoop.hbase.io.HalfStoreFileReader$1.seekTo(HalfStoreFileReader.java:187) at org.apache.hadoop.hbase.io.HalfStoreFileReader.getFirstKey(HalfStoreFileReader.java:348) at org.apache.hadoop.hbase.regionserver.HStoreFile.open(HStoreFile.java:477) at org.apache.hadoop.hbase.regionserver.HStoreFile.initReader(HStoreFile.java:488) at org.apache.hadoop.hbase.regionserver.HStore.createStoreFileAndReader(HStore.java:724) at org.apache.hadoop.hbase.regionserver.HStore.lambda$openStoreFiles$1(HStore.java:574) ... 6 more {code} The region fail opened in one regionserver, then success opened in another regionserver. was (Author: aoxiang): {code} 2019-05-17 12:42:53,319 WARN [RS_OPEN_REGION-regionserver/hb-mbasedata-3:16020-0] regionserver.HRegion: Failed initialize of region=mk_usr_trce,6221a8e3c6a-34f4-4705-9d30-1dc3f235cabc155793,1558068172237.819b2a84f5de78ed3d6ec10061538d6c., starting to roll back memstore java.io.IOException: java.io.IOException: java.lang.NullPointerException at org.apache.hadoop.hbase.regionserver.HRegion.initializeStores(HRegion.java:1106) at org.apache.hadoop.hbase.regionserver.HRegion.initializeStores(HRegion.java:1048) at org.apache.hadoop.hbase.regionserver.HRegion.initializeRegionInternals(HRegion.java:959) at org.apache.hadoop.hbase.regionserver.HRegion.initialize(HRegion.java:915) at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7293) at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7252) at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7224) at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7182) at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7133) at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:133) at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: java.io.IOException: java.lang.NullPointerException at org.apache.hadoop.hbase.regionserver.HStore.openStoreFiles(HStore.java:593) at org.apache.hadoop.hbase.regionserver.HStore.loadStoreFiles(HStore.java:557) at org.apache.hadoop.hbase.regionserver.HStore.<init>(HStore.java:303) at org.apache.hadoop.hbase.regionserver.HRegion.instantiateHStore(HRegion.java:5778) at org.apache.hadoop.hbase.regionserver.HRegion$1.call(HRegion.java:1070) at org.apache.hadoop.hbase.regionserver.HRegion$1.call(HRegion.java:1067) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) ... 3 more Caused by: java.lang.NullPointerException at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.returnBlock(BucketCache.java:1665) at org.apache.hadoop.hbase.io.hfile.BlockCacheUtil.shouldReplaceExistingCacheBlock(BlockCacheUtil.java:250) at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.cacheBlockWithWait(BucketCache.java:426) at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.cacheBlock(BucketCache.java:412) at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.cacheBlock(CombinedBlockCache.java:67) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.lambda$readBlock$2(HFileReaderImpl.java:1501) at java.util.Optional.ifPresent(Optional.java:159) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1499) at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$CellBasedKeyBlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:340) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.seekTo(HFileReaderImpl.java:856) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.seekTo(HFileReaderImpl.java:806) at org.apache.hadoop.hbase.io.HalfStoreFileReader$1.seekTo(HalfStoreFileReader.java:187) at org.apache.hadoop.hbase.io.HalfStoreFileReader.getFirstKey(HalfStoreFileReader.java:348) at org.apache.hadoop.hbase.regionserver.HStoreFile.open(HStoreFile.java:477) at org.apache.hadoop.hbase.regionserver.HStoreFile.initReader(HStoreFile.java:488) at org.apache.hadoop.hbase.regionserver.HStore.createStoreFileAndReader(HStore.java:724) at org.apache.hadoop.hbase.regionserver.HStore.lambda$openStoreFiles$1(HStore.java:574) ... 6 more {code} > Corrupt hfile data > ------------------ > > Key: HBASE-22433 > URL: https://issues.apache.org/jira/browse/HBASE-22433 > Project: HBase > Issue Type: Bug > Affects Versions: 2.2.0 > Reporter: binlijin > Priority: Critical > > We use 2.2.0 version and encounter corrupt cell data. > {code} > 2019-05-15 22:53:59,354 ERROR > [regionserver/hb-mbasedata-14:16020-longCompactions-1557048533421] > regionserver.CompactSplit: Compaction failed > region=mktdm_id_src,99999990,1557681762973.255e9adde013e370deb595c59a7285c3., > storeName=o, priority=196, startTime=1557931927314 > java.lang.IllegalStateException: Invalid currKeyLen 1700752997 or > currValueLen 2002739568. Block offset: 70452918, block length: 66556, > position: 42364 (without header). > at > org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.checkKeyValueLen(HFileReaderImpl.java:1182) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readKeyValueLen(HFileReaderImpl.java:628) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl._next(HFileReaderImpl.java:1080) > at > org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.next(HFileReaderImpl.java:1097) > at > org.apache.hadoop.hbase.regionserver.StoreFileScanner.next(StoreFileScanner.java:208) > at > org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:120) > at > org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:644) > at > org.apache.hadoop.hbase.regionserver.compactions.Compactor.performCompaction(Compactor.java:386) > at > org.apache.hadoop.hbase.regionserver.compactions.Compactor.compact(Compactor.java:326) > at > org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.compact(DefaultCompactor.java:65) > at > org.apache.hadoop.hbase.regionserver.DefaultStoreEngine$DefaultCompactionContext.compact(DefaultStoreEngine.java:126) > at org.apache.hadoop.hbase.regionserver.HStore.compact(HStore.java:1429) > at org.apache.hadoop.hbase.regionserver.HRegion.compact(HRegion.java:2231) > at > org.apache.hadoop.hbase.regionserver.CompactSplit$CompactionRunner.doCompaction(CompactSplit.java:629) > at > org.apache.hadoop.hbase.regionserver.CompactSplit$CompactionRunner.run(CompactSplit.java:671) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) > at java.lang.Thread.run(Thread.java:748) > > 2019-05-15 23:14:24,143 ERROR > [regionserver/hb-mbasedata-14:16020-longCompactions-1557048533422] > regionserver.CompactSplit: Compaction failed > region=mktdm_id_src,9fdee4,1557681762973.1782aebb83eae551e7bdfc2bfa13eb3d., > storeName=o, priority=194, startTime=1557932726849 > java.lang.RuntimeException: Unknown code 98 > at org.apache.hadoop.hbase.KeyValue$Type.codeToType(KeyValue.java:274) > at org.apache.hadoop.hbase.CellUtil.getCellKeyAsString(CellUtil.java:1307) > at > org.apache.hadoop.hbase.io.hfile.HFileWriterImpl.getMidpoint(HFileWriterImpl.java:383) > at > org.apache.hadoop.hbase.io.hfile.HFileWriterImpl.finishBlock(HFileWriterImpl.java:343) > at > org.apache.hadoop.hbase.io.hfile.HFileWriterImpl.close(HFileWriterImpl.java:603) > at > org.apache.hadoop.hbase.regionserver.StoreFileWriter.close(StoreFileWriter.java:376) > at > org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.abortWriter(DefaultCompactor.java:98) > at > org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.abortWriter(DefaultCompactor.java:42) > at > org.apache.hadoop.hbase.regionserver.compactions.Compactor.compact(Compactor.java:335) > at > org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.compact(DefaultCompactor.java:65) > at > org.apache.hadoop.hbase.regionserver.DefaultStoreEngine$DefaultCompactionContext.compact(DefaultStoreEngine.java:126) > at org.apache.hadoop.hbase.regionserver.HStore.compact(HStore.java:1429) > at org.apache.hadoop.hbase.regionserver.HRegion.compact(HRegion.java:2231) > at > org.apache.hadoop.hbase.regionserver.CompactSplit$CompactionRunner.doCompaction(CompactSplit.java:629) > at > org.apache.hadoop.hbase.regionserver.CompactSplit$CompactionRunner.run(CompactSplit.java:671) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) > at java.lang.Thread.run(Thread.java:748) > {code} -- This message was sent by Atlassian JIRA (v7.6.3#76005)