[
https://issues.apache.org/jira/browse/HADOOP-15033?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16256722#comment-16256722
]
Dmitry Chuyko commented on HADOOP-15033:
----------------------------------------
Results on latest JDK 10 on 8 core ARM64 including native library:
aarch64, 8 cores, JDK 10
java.version = 10-internal
java.runtime.name = OpenJDK Runtime Environment
java.runtime.version = 10-internal+0-adhoc.root.hs10
java.vm.version = 10-internal+0-adhoc.root.hs10
java.vm.vendor = Oracle Corporation
java.vm.name = OpenJDK 64-Bit Server VM
java.vm.specification.version = 10
java.specification.version = 10
os.arch = aarch64
os.name = Linux
os.version = 4.9.23-std-1
Data Length = 64 MB
Trials = 5
Direct Buffer Performance Table (bpc: byte-per-crc in MB/sec; #T: #Theads)
| bpc | #T || Zip || ZipC | % diff || PureJava | % diff || PureJavaC
| % diff || Native | % diff || NativeC | % diff |
| 32 | 1 | 168.5 | 182.7 | 8.4% | 101.3 | -44.6% | 93.5
| -7.7% | 455.8 | 387.3% | 465.6 | 2.1% |
| 32 | 2 | 173.3 | 179.7 | 3.7% | 101.6 | -43.5% | 96.4
| -5.1% | 396.0 | 310.9% | 421.8 | 6.5% |
| 32 | 4 | 165.3 | 164.9 | -0.3% | 98.3 | -40.4% | 92.8
| -5.6% | 393.3 | 323.7% | 389.9 | -0.9% |
| 32 | 8 | 164.3 | 163.4 | -0.5% | 98.3 | -39.9% | 92.5
| -5.9% | 395.6 | 327.8% | 391.9 | -0.9% |
| 32 | 16 | 87.9 | 87.5 | -0.4% | 50.8 | -42.0% | 48.3
| -4.9% | 230.6 | 377.7% | 238.1 | 3.2% |
| bpc | #T || Zip || ZipC | % diff || PureJava | % diff || PureJavaC
| % diff || Native | % diff || NativeC | % diff |
| 64 | 1 | 253.6 | 253.7 | 0.0% | 121.9 | -52.0% | 116.5
| -4.4% | 460.3 | 295.2% | 458.3 | -0.4% |
| 64 | 2 | 238.9 | 237.9 | -0.4% | 119.5 | -49.8% | 115.6
| -3.2% | 440.2 | 280.6% | 437.8 | -0.5% |
| 64 | 4 | 239.2 | 239.2 | 0.0% | 119.8 | -49.9% | 115.6
| -3.5% | 438.3 | 279.1% | 477.8 | 9.0% |
| 64 | 8 | 237.2 | 236.0 | -0.5% | 116.3 | -50.7% | 115.0
| -1.1% | 458.1 | 298.3% | 440.1 | -3.9% |
| 64 | 16 | 129.0 | 128.4 | -0.4% | 62.2 | -51.6% | 60.2
| -3.2% | 261.5 | 334.3% | 257.5 | -1.5% |
| bpc | #T || Zip || ZipC | % diff || PureJava | % diff || PureJavaC
| % diff || Native | % diff || NativeC | % diff |
| 128 | 1 | 604.0 | 616.3 | 2.0% | 179.8 | -70.8% | 181.1
| 0.8% | 480.6 | 165.3% | 484.3 | 0.8% |
| 128 | 2 | 606.7 | 608.8 | 0.4% | 180.1 | -70.4% | 180.7
| 0.4% | 462.4 | 155.9% | 461.2 | -0.3% |
| 128 | 4 | 608.9 | 599.3 | -1.6% | 180.3 | -69.9% | 180.3
| 0.0% | 461.4 | 155.8% | 462.6 | 0.3% |
| 128 | 8 | 596.1 | 602.2 | 1.0% | 178.4 | -70.4% | 179.5
| 0.6% | 466.3 | 159.8% | 462.2 | -0.9% |
| 128 | 16 | 357.8 | 349.6 | -2.3% | 92.9 | -73.4% | 93.5
| 0.6% | 277.4 | 196.7% | 284.6 | 2.6% |
| bpc | #T || Zip || ZipC | % diff || PureJava | % diff || PureJavaC
| % diff || Native | % diff || NativeC | % diff |
| 256 | 1 | 793.2 | 794.3 | 0.1% | 200.7 | -74.7% | 209.0
| 4.1% | 533.3 | 155.2% | 485.6 | -8.9% |
| 256 | 2 | 802.6 | 770.6 | -4.0% | 199.0 | -74.2% | 205.3
| 3.1% | 645.8 | 214.6% | 576.9 | -10.7% |
| 256 | 4 | 844.8 | 802.7 | -5.0% | 199.7 | -75.1% | 195.4
| -2.1% | 556.0 | 184.5% | 513.9 | -7.6% |
| 256 | 8 | 778.8 | 760.2 | -2.4% | 195.4 | -74.3% | 202.6
| 3.7% | 503.7 | 148.7% | 483.8 | -4.0% |
| 256 | 16 | 434.1 | 459.8 | 5.9% | 106.1 | -76.9% | 107.9
| 1.7% | 287.8 | 166.7% | 279.6 | -2.8% |
| bpc | #T || Zip || ZipC | % diff || PureJava | % diff || PureJavaC
| % diff || Native | % diff || NativeC | % diff |
| 512 | 1 | 796.8 | 805.1 | 1.0% | 199.8 | -75.2% | 209.6
| 4.9% | 506.7 | 141.7% | 566.2 | 11.7% |
| 512 | 2 | 777.7 | 883.4 | 13.6% | 202.1 | -77.1% | 211.9
| 4.9% | 651.2 | 207.3% | 653.5 | 0.4% |
| 512 | 4 | 849.1 | 858.6 | 1.1% | 198.2 | -76.9% | 212.2
| 7.1% | 618.4 | 191.4% | 558.3 | -9.7% |
| 512 | 8 | 793.7 | 813.7 | 2.5% | 195.1 | -76.0% | 208.2
| 6.7% | 539.7 | 159.2% | 537.8 | -0.4% |
| 512 | 16 | 506.5 | 473.8 | -6.5% | 107.5 | -77.3% | 106.4
| -1.0% | 305.6 | 187.0% | 282.5 | -7.6% |
| bpc | #T || Zip || ZipC | % diff || PureJava | % diff || PureJavaC
| % diff || Native | % diff || NativeC | % diff |
| 1024 | 1 | 925.7 | 820.6 | -11.4% | 204.1 | -75.1% | 215.4
| 5.5% | 509.1 | 136.4% | 506.9 | -0.4% |
| 1024 | 2 | 805.8 | 804.0 | -0.2% | 203.1 | -74.7% | 214.6
| 5.7% | 608.4 | 183.5% | 609.5 | 0.2% |
| 1024 | 4 | 796.5 | 805.1 | 1.1% | 202.5 | -74.8% | 213.8
| 5.6% | 484.7 | 126.7% | 480.4 | -0.9% |
| 1024 | 8 | 857.5 | 889.4 | 3.7% | 202.4 | -77.2% | 218.7
| 8.1% | 619.5 | 183.2% | 616.6 | -0.5% |
| 1024 | 16 | 498.0 | 463.5 | -6.9% | 106.2 | -77.1% | 112.0
| 5.4% | 297.5 | 165.7% | 314.2 | 5.6% |
| bpc | #T || Zip || ZipC | % diff || PureJava | % diff || PureJavaC
| % diff || Native | % diff || NativeC | % diff |
| 2048 | 1 | 947.2 | 861.2 | -9.1% | 205.7 | -76.1% | 219.5
| 6.7% | 505.5 | 130.3% | 508.4 | 0.6% |
| 2048 | 2 | 833.6 | 837.0 | 0.4% | 204.7 | -75.5% | 218.5
| 6.8% | 488.8 | 123.7% | 482.6 | -1.3% |
| 2048 | 4 | 835.6 | 807.5 | -3.4% | 204.9 | -74.6% | 217.8
| 6.3% | 487.5 | 123.8% | 484.9 | -0.5% |
| 2048 | 8 | 820.0 | 802.9 | -2.1% | 203.0 | -74.7% | 217.8
| 7.3% | 481.1 | 120.9% | 478.1 | -0.6% |
| 2048 | 16 | 550.9 | 463.9 | -15.8% | 110.7 | -76.1% | 117.1
| 5.8% | 276.8 | 136.5% | 286.3 | 3.4% |
| bpc | #T || Zip || ZipC | % diff || PureJava | % diff || PureJavaC
| % diff || Native | % diff || NativeC | % diff |
| 4096 | 1 | 861.1 | 869.5 | 1.0% | 206.6 | -76.2% | 221.8
| 7.3% | 509.9 | 129.9% | 507.5 | -0.5% |
| 4096 | 2 | 841.4 | 944.1 | 12.2% | 206.1 | -78.2% | 219.8
| 6.6% | 483.3 | 119.9% | 484.9 | 0.3% |
| 4096 | 4 | 959.8 | 852.0 | -11.2% | 205.9 | -75.8% | 220.1
| 6.9% | 539.6 | 145.1% | 484.4 | -10.2% |
| 4096 | 8 | 843.5 | 838.0 | -0.7% | 205.4 | -75.5% | 219.4
| 6.8% | 488.2 | 122.5% | 489.7 | 0.3% |
| 4096 | 16 | 503.9 | 521.1 | 3.4% | 109.8 | -78.9% | 115.3
| 5.0% | 300.1 | 160.3% | 286.8 | -4.5% |
| bpc | #T || Zip || ZipC | % diff || PureJava | % diff || PureJavaC
| % diff || Native | % diff || NativeC | % diff |
| 8192 | 1 | 888.7 | 884.6 | -0.5% | 208.3 | -76.5% | 223.3
| 7.2% | 510.2 | 128.5% | 506.2 | -0.8% |
| 8192 | 2 | 858.9 | 865.2 | 0.7% | 207.0 | -76.1% | 220.8
| 6.7% | 482.4 | 118.5% | 671.7 | 39.2% |
| 8192 | 4 | 1009.1 | 980.9 | -2.8% | 207.1 | -78.9% | 228.4
| 10.3% | 650.4 | 184.8% | 656.5 | 0.9% |
| 8192 | 8 | 985.1 | 949.5 | -3.6% | 212.2 | -77.6% | 220.6
| 3.9% | 633.6 | 187.2% | 622.2 | -1.8% |
| 8192 | 16 | 587.0 | 583.9 | -0.5% | 107.0 | -81.7% | 116.3
| 8.7% | 321.3 | 176.2% | 316.7 | -1.4% |
| bpc | #T || Zip || ZipC | % diff || PureJava | % diff || PureJavaC
| % diff || Native | % diff || NativeC | % diff |
| 16384 | 1 | 875.3 | 980.9 | 12.1% | 207.5 | -78.8% | 220.6
| 6.3% | 582.7 | 164.1% | 510.0 | -12.5% |
| 16384 | 2 | 1017.4 | 1027.4 | 1.0% | 210.2 | -79.5% | 223.4
| 6.3% | 662.2 | 196.4% | 654.4 | -1.2% |
| 16384 | 4 | 1011.9 | 979.5 | -3.2% | 208.5 | -78.7% | 218.5
| 4.8% | 648.2 | 196.6% | 661.6 | 2.1% |
| 16384 | 8 | 973.3 | 956.2 | -1.8% | 209.5 | -78.1% | 219.0
| 4.6% | 594.5 | 171.4% | 511.9 | -13.9% |
| 16384 | 16 | 534.8 | 496.0 | -7.3% | 107.5 | -78.3% | 115.4
| 7.3% | 280.3 | 142.9% | 285.2 | 1.8% |
| bpc | #T || Zip || ZipC | % diff || PureJava | % diff || PureJavaC
| % diff || Native | % diff || NativeC | % diff |
| 32768 | 1 | 806.5 | 801.0 | -0.7% | 201.9 | -74.8% | 215.5
| 6.7% | 509.4 | 136.4% | 507.4 | -0.4% |
| 32768 | 2 | 882.4 | 824.2 | -6.6% | 201.5 | -75.6% | 213.9
| 6.2% | 483.8 | 126.2% | 486.6 | 0.6% |
| 32768 | 4 | 777.9 | 781.5 | 0.5% | 201.1 | -74.3% | 214.2
| 6.5% | 483.6 | 125.8% | 481.3 | -0.5% |
| 32768 | 8 | 782.8 | 781.5 | -0.2% | 200.1 | -74.4% | 206.4
| 3.2% | 482.6 | 133.8% | 490.3 | 1.6% |
| 32768 | 16 | 460.5 | 476.0 | 3.4% | 107.0 | -77.5% | 116.8
| 9.2% | 294.4 | 152.0% | 299.8 | 1.8% |
| bpc | #T || Zip || ZipC | % diff || PureJava | % diff || PureJavaC
| % diff || Native | % diff || NativeC | % diff |
| 65536 | 1 | 801.1 | 806.1 | 0.6% | 202.1 | -74.9% | 215.4
| 6.6% | 509.1 | 136.3% | 509.6 | 0.1% |
| 65536 | 2 | 782.8 | 887.4 | 13.4% | 201.0 | -77.3% | 214.7
| 6.8% | 482.9 | 125.0% | 485.0 | 0.4% |
| 65536 | 4 | 775.1 | 786.4 | 1.4% | 205.9 | -73.8% | 208.1
| 1.1% | 662.3 | 218.3% | 662.2 | -0.0% |
| 65536 | 8 | 889.3 | 875.5 | -1.5% | 204.8 | -76.6% | 216.1
| 5.5% | 507.1 | 134.7% | 547.1 | 7.9% |
| 65536 | 16 | 468.3 | 458.6 | -2.1% | 106.3 | -76.8% | 114.0
| 7.2% | 288.8 | 153.3% | 308.7 | 6.9% |
Please note that current JDK implementation is better than even the one in
native library for both checksums.
> Use java.util.zip.CRC32C for Java 9 and above
> ---------------------------------------------
>
> Key: HADOOP-15033
> URL: https://issues.apache.org/jira/browse/HADOOP-15033
> Project: Hadoop Common
> Issue Type: Improvement
> Components: performance, util
> Affects Versions: 3.0.0
> Reporter: Dmitry Chuyko
>
> java.util.zip.CRC32C implementation is available since Java 9.
> https://docs.oracle.com/javase/9/docs/api/java/util/zip/CRC32C.html
> Platform specific assembler intrinsics make it more effective than any pure
> Java implementation.
> Hadoop is compiled against Java 8 but class constructor may be accessible
> with method handle on 9 to instances implementing Checksum in runtime.
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-issues-h...@hadoop.apache.org
