[
https://issues.apache.org/jira/browse/HDFS-1605?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Liang Xie updated HDFS-1605:
----------------------------
Attachment: HDFS-1605.txt
Attached is a patch against trunk.
The getFileLength hotspot pattern like below:
"IPC Server handler 88 on 12600" daemon prio=10 tid=0x00007f82fc241580
nid=0x4ddc waiting for monitor entry [0x00007f821eefb000]
java.lang.Thread.State: BLOCKED (on object monitor)
at org.apache.hadoop.hdfs.DFSInputStream.getFileLength(DFSInputStream.java:242)
- waiting to lock <0x00000004404597a8> (a org.apache.hadoop.hdfs.DFSInputStream)
at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:982)
at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:73)
at
org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1393)
at
org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderV2.readBlockDataInternal(HFileBlock.java:1829)
at
org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderV2.readBlockData(HFileBlock.java:1673)
at
org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:341)
at
org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:254)
at
org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:485)
at
org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:506)
at
org.apache.hadoop.hbase.regionserver.StoreFileScanner.seekAtOrAfter(StoreFileScanner.java:226)
at
org.apache.hadoop.hbase.regionserver.StoreFileScanner.seek(StoreFileScanner.java:146)
at
org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:354)
at
org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:385)
at
org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:344)
at
org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:304)
at
org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:584)
- locked <0x00000004285cb478> (a
org.apache.hadoop.hbase.regionserver.StoreScanner)
at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:446)
- locked <0x00000004285cb478> (a
org.apache.hadoop.hbase.regionserver.StoreScanner)
at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:164)
at
org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:3658)
at
org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:3590)
- locked <0x00000004285cb310> (a
org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
at
org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:3615)
- locked <0x00000004285cb310> (a
org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
at
org.apache.hadoop.hbase.regionserver.HRegionServer.next(HRegionServer.java:2513)
at sun.reflect.GeneratedMethodAccessor28.invoke(Unknown Source)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at
org.apache.hadoop.hbase.ipc.SecureRpcEngine$Server.call(SecureRpcEngine.java:460)
at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1457)
"IPC Server handler 192 on 12600" daemon prio=10 tid=0x00007f82fc2dd910
nid=0x4e44 runnable [0x00007f821d493000]
java.lang.Thread.State: RUNNABLE
at java.io.FileInputStream.readBytes(Native Method)
at java.io.FileInputStream.read(FileInputStream.java:220)
at org.apache.hadoop.hdfs.BlockReaderLocal.read(BlockReaderLocal.java:568)
- locked <0x00000004287fb350> (a org.apache.hadoop.hdfs.BlockReaderLocal)
at
org.apache.hadoop.hdfs.DFSInputStream$ByteArrayStrategy.doRead(DFSInputStream.java:542)
at org.apache.hadoop.hdfs.DFSInputStream.readBuffer(DFSInputStream.java:594)
- locked <0x00000004404597a8> (a org.apache.hadoop.hdfs.DFSInputStream)
at
org.apache.hadoop.hdfs.DFSInputStream.readWithStrategy(DFSInputStream.java:648)
at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:689)
- locked <0x00000004404597a8> (a org.apache.hadoop.hdfs.DFSInputStream)
at java.io.DataInputStream.read(DataInputStream.java:132)
at
org.apache.hadoop.hbase.io.hfile.HFileBlock.readWithExtra(HFileBlock.java:614)
at
org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1384)
at
org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderV2.readBlockDataInternal(HFileBlock.java:1829)
at
org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderV2.readBlockData(HFileBlock.java:1673)
at
org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:341)
at
org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:254)
at
org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:485)
at
org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:506)
at
org.apache.hadoop.hbase.regionserver.StoreFileScanner.seekAtOrAfter(StoreFileScanner.java:226)
at
org.apache.hadoop.hbase.regionserver.StoreFileScanner.seek(StoreFileScanner.java:146)
at
org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:354)
at
org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:385)
at
org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:344)
at
org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:304)
at
org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:584)
- locked <0x0000000428374d70> (a
org.apache.hadoop.hbase.regionserver.StoreScanner)
at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:446)
- locked <0x0000000428374d70> (a
org.apache.hadoop.hbase.regionserver.StoreScanner)
at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:164)
at
org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:3658)
at
org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:3590)
- locked <0x0000000428374c08> (a
org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
at
org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:3615)
- locked <0x0000000428374c08> (a
org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
at
org.apache.hadoop.hbase.regionserver.HRegionServer.next(HRegionServer.java:2513)
at sun.reflect.GeneratedMethodAccessor28.invoke(Unknown Source)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at
org.apache.hadoop.hbase.ipc.SecureRpcEngine$Server.call(SecureRpcEngine.java:460)
at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1457)
> Convert DFSInputStream synchronized sections to a ReadWrite lock
> ----------------------------------------------------------------
>
> Key: HDFS-1605
> URL: https://issues.apache.org/jira/browse/HDFS-1605
> Project: Hadoop HDFS
> Issue Type: Improvement
> Components: hdfs-client
> Reporter: dhruba borthakur
> Assignee: dhruba borthakur
> Attachments: DFSClientRWlock.1.txt, DFSClientRWlock.3.txt,
> HDFS-1605.txt
>
>
> Hbase does concurrent preads from multiple threads to different blocks of the
> same hdfs file. Each of these pread calls invoke
> DFSInputStream.getFileLength() and DFSInputStream.getBlockAt(). These methods
> are "synchronized", thus causing all the concurrent threads to serialize. It
> would help performance to convert this to a Read/Write lock
--
This message was sent by Atlassian JIRA
(v6.1.4#6159)
