[
https://issues.apache.org/jira/browse/HBASE-20236?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16418472#comment-16418472
]
stack commented on HBASE-20236:
-------------------------------
Hmm... No. Semaphore is slightly faster (3-4%) even though it has this
crazy-looking locking profile:
{code}
1 Started [lock] profiling
2 --- Execution profile ---
3 Total samples: 2388092
4 Skipped: 608 (0.03%)
5
6 Frame buffer usage: 0.2034%
7
8 Total: 730190050914 (97.47%) samples: 2367078
9 [ 0] java.util.concurrent.Semaphore$NonfairSync
10 [ 1] sun.misc.Unsafe.park
11 [ 2] java.util.concurrent.locks.LockSupport.park
12 [ 3]
java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt
13 [ 4]
java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly
14 [ 5]
java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly
15 [ 6] java.util.concurrent.Semaphore.acquire
16 [ 7]
org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner
17 [ 8] org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run
18
19 Total: 15140093209 (2.02%) samples: 1
20 [ 0] java.lang.Object
21 [ 1] sun.nio.ch.ServerSocketChannelImpl.accept
22 [ 2] org.eclipse.jetty.server.ServerConnector.accept
23 [ 3] org.eclipse.jetty.server.AbstractConnector$Acceptor.run
24 [ 4] org.eclipse.jetty.util.thread.QueuedThreadPool.runJob
25 [ 5] org.eclipse.jetty.util.thread.QueuedThreadPool$2.run
26 [ 6] java.lang.Thread.run
27
28 Total: 1256664266 (0.17%) samples: 4945
29 [ 0] java.util.concurrent.ConcurrentHashMap
30 [ 1]
org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.<init>
31 [ 2]
org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.<init>
32 [ 3]
org.apache.hadoop.hbase.regionserver.ReversedRegionScannerImpl.<init>
33 [ 4]
org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner
34 [ 5] org.apache.hadoop.hbase.regionserver.HRegion.getScanner
35 [ 6] org.apache.hadoop.hbase.regionserver.HRegion.getScanner
36 [ 7] org.apache.hadoop.hbase.regionserver.HRegion.getScanner
37 [ 8] org.apache.hadoop.hbase.regionserver.RSRpcServices.get
38 [ 9] org.apache.hadoop.hbase.regionserver.RSRpcServices.get
39 [10]
org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod
40 [11] org.apache.hadoop.hbase.ipc.RpcServer.call
41 [12] org.apache.hadoop.hbase.ipc.CallRunner.run
42 [13] org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run
43 [14] org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run
44
45 Total: 662087825 (0.09%) samples: 3570
46 [ 0] org.apache.hadoop.hdfs.PeerCache
47 [ 1] com.google.common.collect.LinkedListMultimap.size
48 [ 2] org.apache.hadoop.hdfs.PeerCache.putInternal
49 [ 3] org.apache.hadoop.hdfs.PeerCache.put
50 [ 4] org.apache.hadoop.hdfs.RemoteBlockReader2.close
51 [ 5] org.apache.hadoop.hdfs.DFSInputStream.actualGetFromOneDataNode
52 [ 6] org.apache.hadoop.hdfs.DFSInputStream.fetchBlockByteRange
53 [ 7] org.apache.hadoop.hdfs.DFSInputStream.pread
54 [ 8] org.apache.hadoop.hdfs.DFSInputStream.read
55 [ 9] org.apache.hadoop.fs.FSDataInputStream.read
56 [10] org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra
57 [11]
org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset
58 [12]
org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal
59 [13]
org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData
60 [14] org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock
61 [15]
org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$CellBasedKeyBlockIndexReader.loadDataBlockWithScanInfo
62 [16]
org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.seekTo
63 [17]
org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.seekTo
64 [18] org.apache.hadoop.hbase.regionserver.StoreFileScanner.seekAtOrAfter
65 [19] org.apache.hadoop.hbase.regionserver.StoreFileScanner.seek
66 [20] org.apache.hadoop.hbase.regionserver.StoreScanner.seekScanners
67 [21] org.apache.hadoop.hbase.regionserver.StoreScanner.<init>
68 [22] org.apache.hadoop.hbase.regionserver.HStore.createScanner
69 [23] org.apache.hadoop.hbase.regionserver.HStore.getScanner
70 [24]
org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.initializeScanners
71 [25]
org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.<init>
72 [26]
org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner
73 [27] org.apache.hadoop.hbase.regionserver.HRegion.getScanner
74 [28] org.apache.hadoop.hbase.regionserver.HRegion.getScanner
75 [29] org.apache.hadoop.hbase.regionserver.HRegion.getScanner
76 [30] org.apache.hadoop.hbase.regionserver.RSRpcServices.get
77 [31] org.apache.hadoop.hbase.regionserver.RSRpcServices.get
78 [32]
org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod
79 [33] org.apache.hadoop.hbase.ipc.RpcServer.call
80 [34] org.apache.hadoop.hbase.ipc.CallRunner.run
81 [35] org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run
82 [36] org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run
83
84 Total: 497826090 (0.07%) samples: 3176
85 [ 0] org.apache.hadoop.hdfs.PeerCache
86 [ 1]
com.google.common.collect.LinkedListMultimap$ValueForKeyIterator.remove
87 [ 2] org.apache.hadoop.hdfs.PeerCache.getInternal
88 [ 3] org.apache.hadoop.hdfs.PeerCache.get
89 [ 4] org.apache.hadoop.hdfs.BlockReaderFactory.nextTcpPeer
90 [ 5]
org.apache.hadoop.hdfs.BlockReaderFactory.getRemoteBlockReaderFromTcp
91 [ 6] org.apache.hadoop.hdfs.BlockReaderFactory.build
92 [ 7] org.apache.hadoop.hdfs.DFSInputStream.actualGetFromOneDataNode
93 [ 8] org.apache.hadoop.hdfs.DFSInputStream.fetchBlockByteRange
94 [ 9] org.apache.hadoop.hdfs.DFSInputStream.pread
95 [10] org.apache.hadoop.hdfs.DFSInputStream.read
96 [11] org.apache.hadoop.fs.FSDataInputStream.read
97 [12] org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra
98 [13]
org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset
99 [14]
org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal
100 [15]
org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData
101 [16] org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock
102 [17]
org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$CellBasedKeyBlockIndexReader.loadDataBlockWithScanInfo
103 [18]
org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.seekTo
104 [19]
org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.seekTo
105 [20] org.apache.hadoop.hbase.regionserver.StoreFileScanner.seekAtOrAfter
...{code}
YCSB numbers:
{
"metric" : "OVERALL",
"measurement" : "RunTime(ms)",
"value" : 1200122
}, {
"metric" : "OVERALL",
"measurement" : "Throughput(ops/sec)",
"value" : 81001.45818508451
}
Let me attach pictures.
> [locking] Write-time worst offenders
> ------------------------------------
>
> Key: HBASE-20236
> URL: https://issues.apache.org/jira/browse/HBASE-20236
> Project: HBase
> Issue Type: Sub-task
> Components: Performance
> Affects Versions: 2.0.0-beta-2
> Reporter: stack
> Priority: Major
> Attachments: no_semaphore_vs_semaphore.png
>
>
> Messing w/ my new toy, here are worst offenders locking; they must be bad if
> they show up in this sampling profiler:
> {code}
> 7 Total: 769321884622 (99.24%) samples: 2965
> 8 [ 0] java.util.concurrent.Semaphore$NonfairSync
> 9 [ 1] sun.misc.Unsafe.park
> 10 [ 2] java.util.concurrent.locks.LockSupport.park
> 11 [ 3]
> java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt
> 12 [ 4]
> java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly
> 13 [ 5]
> java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly
> 14 [ 6] java.util.concurrent.Semaphore.acquire
> 15 [ 7]
> org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner
> 16 [ 8] org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run
> 17
> 18 Total: 4284274263 (0.55%) samples: 23543
> 19 [ 0] org.apache.hadoop.hbase.regionserver.MutableSegment
> 20 [ 1] org.apache.hadoop.hbase.ByteBufferKeyValue.getSequenceId
> 21 [ 2] org.apache.hadoop.hbase.regionserver.Segment.updateMetaInfo
> 22 [ 3] org.apache.hadoop.hbase.regionserver.Segment.internalAdd
> 23 [ 4] org.apache.hadoop.hbase.regionserver.MutableSegment.add
> 24 [ 5] org.apache.hadoop.hbase.regionserver.AbstractMemStore.internalAdd
> 25 [ 6] org.apache.hadoop.hbase.regionserver.AbstractMemStore.add
> 26 [ 7] org.apache.hadoop.hbase.regionserver.AbstractMemStore.add
> 27 [ 8] org.apache.hadoop.hbase.regionserver.HStore.add
> 28 [ 9] org.apache.hadoop.hbase.regionserver.HRegion.applyToMemStore
> 29 [10] org.apache.hadoop.hbase.regionserver.HRegion.access$600
> 30 [11]
> org.apache.hadoop.hbase.regionserver.HRegion$BatchOperation.applyFamilyMapToMemStore
> 31 [12]
> org.apache.hadoop.hbase.regionserver.HRegion$BatchOperation.lambda$writeMiniBatchOperationsToMemStore$0
> 32 [13]
> org.apache.hadoop.hbase.regionserver.HRegion$BatchOperation$$Lambda$442.1445825895.visit
> 33 [14]
> org.apache.hadoop.hbase.regionserver.HRegion$BatchOperation.visitBatchOperations
> 34 [15]
> org.apache.hadoop.hbase.regionserver.HRegion$BatchOperation.writeMiniBatchOperationsToMemStore
> 35 [16]
> org.apache.hadoop.hbase.regionserver.HRegion$MutationBatchOperation.writeMiniBatchOperationsToMemStore
> 36 [17] org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutate
> 37 [18] org.apache.hadoop.hbase.regionserver.HRegion.batchMutate
> 38 [19] org.apache.hadoop.hbase.regionserver.HRegion.batchMutate
> 39 [20] org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp
> 40 [21]
> org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicBatchOp
> 41 [22]
> org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation
> 42 [23] org.apache.hadoop.hbase.regionserver.RSRpcServices.multi
> 43 [24]
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod
> 44 [25] org.apache.hadoop.hbase.ipc.RpcServer.call
> 45 [26] org.apache.hadoop.hbase.ipc.CallRunner.run
> 46 [27] org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run
> 47 [28] org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run
> 48
> 49 Total: 717708856 (0.09%) samples: 214
> 50 [ 0] java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync
> 51 [ 1] sun.misc.Unsafe.park
> 52 [ 2] java.util.concurrent.locks.LockSupport.park
> 53 [ 3]
> java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt
> 54 [ 4] java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued
> 55 [ 5] java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire
> 56 [ 6] java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock.lock
> 57 [ 7] org.apache.hadoop.hbase.regionserver.HRegion.blockUpdates
> 58 [ 8]
> org.apache.hadoop.hbase.regionserver.RegionServicesForStores.blockUpdates
> 59 [ 9]
> org.apache.hadoop.hbase.regionserver.CompactingMemStore.flushInMemory
> 60 [10]
> org.apache.hadoop.hbase.regionserver.CompactingMemStore$InMemoryFlushRunnable.run
> 61 [11] java.util.concurrent.ThreadPoolExecutor.runWorker
> 62 [12] java.util.concurrent.ThreadPoolExecutor$Worker.run
> 63 [13] java.lang.Thread.run
> ...
> {code}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
