Apache9 commented on PR #4596: URL: https://github.com/apache/hbase/pull/4596#issuecomment-1175618564
> I think we have some other issues with the server side changes, I see multiple compaction failures: > > ``` > 2022-07-05 19:00:06,828 WARN [20-longCompactions-0] hdfs.DFSClient - Failed to connect to dn1/10.118.172.92:50010 for block BP-1698803826-10.118.165.103-1656451448670:blk_1074099523_359025, add to deadNodes and continue. > java.net.ConnectException: Connection refused > at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) > at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) > at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206) > at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:539) > at org.apache.hadoop.hdfs.DFSClient.newConnectedPeer(DFSClient.java:2913) > at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.nextTcpPeer(BlockReaderFactory.java:851) > at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.getRemoteBlockReaderFromTcp(BlockReaderFactory.java:753) > at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.build(BlockReaderFactory.java:387) > at org.apache.hadoop.hdfs.DFSInputStream.getBlockReader(DFSInputStream.java:843) > at org.apache.hadoop.hdfs.DFSInputStream.blockSeekTo(DFSInputStream.java:772) > at org.apache.hadoop.hdfs.DFSInputStream.seekToBlockSource(DFSInputStream.java:1825) > at org.apache.hadoop.hdfs.DFSInputStream.readBuffer(DFSInputStream.java:1035) > at org.apache.hadoop.hdfs.DFSInputStream.readWithStrategy(DFSInputStream.java:1074) > at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:1138) > at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:148) > at org.apache.hadoop.hbase.io.util.BlockIOUtils.readWithExtra(BlockIOUtils.java:180) > at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1450) > at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1681) > at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1492) > at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1308) > at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:739) > at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1480) > at org.apache.hadoop.hbase.io.HalfStoreFileReader$1.next(HalfStoreFileReader.java:140) > at org.apache.hadoop.hbase.regionserver.StoreFileScanner.next(StoreFileScanner.java:194) > at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:112) > at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:677) > at org.apache.hadoop.hbase.regionserver.compactions.Compactor.performCompaction(Compactor.java:417) > at org.apache.hadoop.hbase.regionserver.compactions.Compactor.compact(Compactor.java:345) > at org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.compact(DefaultCompactor.java:65) > at org.apache.hadoop.hbase.regionserver.DefaultStoreEngine$DefaultCompactionContext.compact(DefaultStoreEngine.java:122) > at org.apache.hadoop.hbase.regionserver.HStore.compact(HStore.java:1464) > at org.apache.hadoop.hbase.regionserver.HRegion.compact(HRegion.java:2286) > at org.apache.hadoop.hbase.regionserver.CompactSplit$CompactionRunner.doCompaction(CompactSplit.java:618) > at org.apache.hadoop.hbase.regionserver.CompactSplit$CompactionRunner.run(CompactSplit.java:666) > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) > at java.lang.Thread.run(Thread.java:750) > ``` > > ``` > 2022-07-05 18:59:43,010 WARN [20-longCompactions-0] impl.BlockReaderFactory - I/O error constructing remote block reader. > java.net.ConnectException: Connection refused > at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) > at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) > at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206) > at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:539) > at org.apache.hadoop.hdfs.DFSClient.newConnectedPeer(DFSClient.java:2913) > at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.nextTcpPeer(BlockReaderFactory.java:851) > at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.getRemoteBlockReaderFromTcp(BlockReaderFactory.java:753) > at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.build(BlockReaderFactory.java:387) > at org.apache.hadoop.hdfs.DFSInputStream.getBlockReader(DFSInputStream.java:843) > at org.apache.hadoop.hdfs.DFSInputStream.blockSeekTo(DFSInputStream.java:772) > at org.apache.hadoop.hdfs.DFSInputStream.seekToBlockSource(DFSInputStream.java:1825) > at org.apache.hadoop.hdfs.DFSInputStream.readBuffer(DFSInputStream.java:1035) > at org.apache.hadoop.hdfs.DFSInputStream.readWithStrategy(DFSInputStream.java:1074) > at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:1138) > at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:148) > at org.apache.hadoop.hbase.io.util.BlockIOUtils.readWithExtra(BlockIOUtils.java:180) > at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1450) > at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1681) > at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1492) > at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1308) > at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:739) > at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1480) > at org.apache.hadoop.hbase.regionserver.StoreFileScanner.next(StoreFileScanner.java:194) > at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:112) > at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:677) > at org.apache.hadoop.hbase.regionserver.compactions.Compactor.performCompaction(Compactor.java:417) > at org.apache.hadoop.hbase.regionserver.compactions.Compactor.compact(Compactor.java:345) > at org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.compact(DefaultCompactor.java:65) > at org.apache.hadoop.hbase.regionserver.DefaultStoreEngine$DefaultCompactionContext.compact(DefaultStoreEngine.java:122) > at org.apache.hadoop.hbase.regionserver.HStore.compact(HStore.java:1464) > at org.apache.hadoop.hbase.regionserver.HRegion.compact(HRegion.java:2286) > at org.apache.hadoop.hbase.regionserver.CompactSplit$CompactionRunner.doCompaction(CompactSplit.java:618) > at org.apache.hadoop.hbase.regionserver.CompactSplit$CompactionRunner.run(CompactSplit.java:666) > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) > at java.lang.Thread.run(Thread.java:750) > ``` > > Multiple BP abandon errors as well: > > ``` > 2022-07-05 18:59:44,696 WARN [Thread-1126] hdfs.DataStreamer - Abandoning BP-1698803826-10.118.165.103-1656451448670:blk_1074101424_360934 > ``` > > Basically HDFS RPC calls are impacted. The changes here do not affect the DFSClient part. Especially, the above failures are in the compaction thread pool, not in the rpc handler threads, which are not affected by the changes here too. Here we just changed the implementation of the decode part in our rpc server implementation when SASL is enabled. So I guess it is just some other problems happen at the same time... -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: issues-unsubscr...@hbase.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
