[ https://issues.apache.org/jira/browse/HADOOP-15033?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16256722#comment-16256722 ]
Dmitry Chuyko commented on HADOOP-15033: ---------------------------------------- Results on latest JDK 10 on 8 core ARM64 including native library: aarch64, 8 cores, JDK 10 java.version = 10-internal java.runtime.name = OpenJDK Runtime Environment java.runtime.version = 10-internal+0-adhoc.root.hs10 java.vm.version = 10-internal+0-adhoc.root.hs10 java.vm.vendor = Oracle Corporation java.vm.name = OpenJDK 64-Bit Server VM java.vm.specification.version = 10 java.specification.version = 10 os.arch = aarch64 os.name = Linux os.version = 4.9.23-std-1 Data Length = 64 MB Trials = 5 Direct Buffer Performance Table (bpc: byte-per-crc in MB/sec; #T: #Theads) | bpc | #T || Zip || ZipC | % diff || PureJava | % diff || PureJavaC | % diff || Native | % diff || NativeC | % diff | | 32 | 1 | 168.5 | 182.7 | 8.4% | 101.3 | -44.6% | 93.5 | -7.7% | 455.8 | 387.3% | 465.6 | 2.1% | | 32 | 2 | 173.3 | 179.7 | 3.7% | 101.6 | -43.5% | 96.4 | -5.1% | 396.0 | 310.9% | 421.8 | 6.5% | | 32 | 4 | 165.3 | 164.9 | -0.3% | 98.3 | -40.4% | 92.8 | -5.6% | 393.3 | 323.7% | 389.9 | -0.9% | | 32 | 8 | 164.3 | 163.4 | -0.5% | 98.3 | -39.9% | 92.5 | -5.9% | 395.6 | 327.8% | 391.9 | -0.9% | | 32 | 16 | 87.9 | 87.5 | -0.4% | 50.8 | -42.0% | 48.3 | -4.9% | 230.6 | 377.7% | 238.1 | 3.2% | | bpc | #T || Zip || ZipC | % diff || PureJava | % diff || PureJavaC | % diff || Native | % diff || NativeC | % diff | | 64 | 1 | 253.6 | 253.7 | 0.0% | 121.9 | -52.0% | 116.5 | -4.4% | 460.3 | 295.2% | 458.3 | -0.4% | | 64 | 2 | 238.9 | 237.9 | -0.4% | 119.5 | -49.8% | 115.6 | -3.2% | 440.2 | 280.6% | 437.8 | -0.5% | | 64 | 4 | 239.2 | 239.2 | 0.0% | 119.8 | -49.9% | 115.6 | -3.5% | 438.3 | 279.1% | 477.8 | 9.0% | | 64 | 8 | 237.2 | 236.0 | -0.5% | 116.3 | -50.7% | 115.0 | -1.1% | 458.1 | 298.3% | 440.1 | -3.9% | | 64 | 16 | 129.0 | 128.4 | -0.4% | 62.2 | -51.6% | 60.2 | -3.2% | 261.5 | 334.3% | 257.5 | -1.5% | | bpc | #T || Zip || ZipC | % diff || PureJava | % diff || PureJavaC | % diff || Native | % diff || NativeC | % diff | | 128 | 1 | 604.0 | 616.3 | 2.0% | 179.8 | -70.8% | 181.1 | 0.8% | 480.6 | 165.3% | 484.3 | 0.8% | | 128 | 2 | 606.7 | 608.8 | 0.4% | 180.1 | -70.4% | 180.7 | 0.4% | 462.4 | 155.9% | 461.2 | -0.3% | | 128 | 4 | 608.9 | 599.3 | -1.6% | 180.3 | -69.9% | 180.3 | 0.0% | 461.4 | 155.8% | 462.6 | 0.3% | | 128 | 8 | 596.1 | 602.2 | 1.0% | 178.4 | -70.4% | 179.5 | 0.6% | 466.3 | 159.8% | 462.2 | -0.9% | | 128 | 16 | 357.8 | 349.6 | -2.3% | 92.9 | -73.4% | 93.5 | 0.6% | 277.4 | 196.7% | 284.6 | 2.6% | | bpc | #T || Zip || ZipC | % diff || PureJava | % diff || PureJavaC | % diff || Native | % diff || NativeC | % diff | | 256 | 1 | 793.2 | 794.3 | 0.1% | 200.7 | -74.7% | 209.0 | 4.1% | 533.3 | 155.2% | 485.6 | -8.9% | | 256 | 2 | 802.6 | 770.6 | -4.0% | 199.0 | -74.2% | 205.3 | 3.1% | 645.8 | 214.6% | 576.9 | -10.7% | | 256 | 4 | 844.8 | 802.7 | -5.0% | 199.7 | -75.1% | 195.4 | -2.1% | 556.0 | 184.5% | 513.9 | -7.6% | | 256 | 8 | 778.8 | 760.2 | -2.4% | 195.4 | -74.3% | 202.6 | 3.7% | 503.7 | 148.7% | 483.8 | -4.0% | | 256 | 16 | 434.1 | 459.8 | 5.9% | 106.1 | -76.9% | 107.9 | 1.7% | 287.8 | 166.7% | 279.6 | -2.8% | | bpc | #T || Zip || ZipC | % diff || PureJava | % diff || PureJavaC | % diff || Native | % diff || NativeC | % diff | | 512 | 1 | 796.8 | 805.1 | 1.0% | 199.8 | -75.2% | 209.6 | 4.9% | 506.7 | 141.7% | 566.2 | 11.7% | | 512 | 2 | 777.7 | 883.4 | 13.6% | 202.1 | -77.1% | 211.9 | 4.9% | 651.2 | 207.3% | 653.5 | 0.4% | | 512 | 4 | 849.1 | 858.6 | 1.1% | 198.2 | -76.9% | 212.2 | 7.1% | 618.4 | 191.4% | 558.3 | -9.7% | | 512 | 8 | 793.7 | 813.7 | 2.5% | 195.1 | -76.0% | 208.2 | 6.7% | 539.7 | 159.2% | 537.8 | -0.4% | | 512 | 16 | 506.5 | 473.8 | -6.5% | 107.5 | -77.3% | 106.4 | -1.0% | 305.6 | 187.0% | 282.5 | -7.6% | | bpc | #T || Zip || ZipC | % diff || PureJava | % diff || PureJavaC | % diff || Native | % diff || NativeC | % diff | | 1024 | 1 | 925.7 | 820.6 | -11.4% | 204.1 | -75.1% | 215.4 | 5.5% | 509.1 | 136.4% | 506.9 | -0.4% | | 1024 | 2 | 805.8 | 804.0 | -0.2% | 203.1 | -74.7% | 214.6 | 5.7% | 608.4 | 183.5% | 609.5 | 0.2% | | 1024 | 4 | 796.5 | 805.1 | 1.1% | 202.5 | -74.8% | 213.8 | 5.6% | 484.7 | 126.7% | 480.4 | -0.9% | | 1024 | 8 | 857.5 | 889.4 | 3.7% | 202.4 | -77.2% | 218.7 | 8.1% | 619.5 | 183.2% | 616.6 | -0.5% | | 1024 | 16 | 498.0 | 463.5 | -6.9% | 106.2 | -77.1% | 112.0 | 5.4% | 297.5 | 165.7% | 314.2 | 5.6% | | bpc | #T || Zip || ZipC | % diff || PureJava | % diff || PureJavaC | % diff || Native | % diff || NativeC | % diff | | 2048 | 1 | 947.2 | 861.2 | -9.1% | 205.7 | -76.1% | 219.5 | 6.7% | 505.5 | 130.3% | 508.4 | 0.6% | | 2048 | 2 | 833.6 | 837.0 | 0.4% | 204.7 | -75.5% | 218.5 | 6.8% | 488.8 | 123.7% | 482.6 | -1.3% | | 2048 | 4 | 835.6 | 807.5 | -3.4% | 204.9 | -74.6% | 217.8 | 6.3% | 487.5 | 123.8% | 484.9 | -0.5% | | 2048 | 8 | 820.0 | 802.9 | -2.1% | 203.0 | -74.7% | 217.8 | 7.3% | 481.1 | 120.9% | 478.1 | -0.6% | | 2048 | 16 | 550.9 | 463.9 | -15.8% | 110.7 | -76.1% | 117.1 | 5.8% | 276.8 | 136.5% | 286.3 | 3.4% | | bpc | #T || Zip || ZipC | % diff || PureJava | % diff || PureJavaC | % diff || Native | % diff || NativeC | % diff | | 4096 | 1 | 861.1 | 869.5 | 1.0% | 206.6 | -76.2% | 221.8 | 7.3% | 509.9 | 129.9% | 507.5 | -0.5% | | 4096 | 2 | 841.4 | 944.1 | 12.2% | 206.1 | -78.2% | 219.8 | 6.6% | 483.3 | 119.9% | 484.9 | 0.3% | | 4096 | 4 | 959.8 | 852.0 | -11.2% | 205.9 | -75.8% | 220.1 | 6.9% | 539.6 | 145.1% | 484.4 | -10.2% | | 4096 | 8 | 843.5 | 838.0 | -0.7% | 205.4 | -75.5% | 219.4 | 6.8% | 488.2 | 122.5% | 489.7 | 0.3% | | 4096 | 16 | 503.9 | 521.1 | 3.4% | 109.8 | -78.9% | 115.3 | 5.0% | 300.1 | 160.3% | 286.8 | -4.5% | | bpc | #T || Zip || ZipC | % diff || PureJava | % diff || PureJavaC | % diff || Native | % diff || NativeC | % diff | | 8192 | 1 | 888.7 | 884.6 | -0.5% | 208.3 | -76.5% | 223.3 | 7.2% | 510.2 | 128.5% | 506.2 | -0.8% | | 8192 | 2 | 858.9 | 865.2 | 0.7% | 207.0 | -76.1% | 220.8 | 6.7% | 482.4 | 118.5% | 671.7 | 39.2% | | 8192 | 4 | 1009.1 | 980.9 | -2.8% | 207.1 | -78.9% | 228.4 | 10.3% | 650.4 | 184.8% | 656.5 | 0.9% | | 8192 | 8 | 985.1 | 949.5 | -3.6% | 212.2 | -77.6% | 220.6 | 3.9% | 633.6 | 187.2% | 622.2 | -1.8% | | 8192 | 16 | 587.0 | 583.9 | -0.5% | 107.0 | -81.7% | 116.3 | 8.7% | 321.3 | 176.2% | 316.7 | -1.4% | | bpc | #T || Zip || ZipC | % diff || PureJava | % diff || PureJavaC | % diff || Native | % diff || NativeC | % diff | | 16384 | 1 | 875.3 | 980.9 | 12.1% | 207.5 | -78.8% | 220.6 | 6.3% | 582.7 | 164.1% | 510.0 | -12.5% | | 16384 | 2 | 1017.4 | 1027.4 | 1.0% | 210.2 | -79.5% | 223.4 | 6.3% | 662.2 | 196.4% | 654.4 | -1.2% | | 16384 | 4 | 1011.9 | 979.5 | -3.2% | 208.5 | -78.7% | 218.5 | 4.8% | 648.2 | 196.6% | 661.6 | 2.1% | | 16384 | 8 | 973.3 | 956.2 | -1.8% | 209.5 | -78.1% | 219.0 | 4.6% | 594.5 | 171.4% | 511.9 | -13.9% | | 16384 | 16 | 534.8 | 496.0 | -7.3% | 107.5 | -78.3% | 115.4 | 7.3% | 280.3 | 142.9% | 285.2 | 1.8% | | bpc | #T || Zip || ZipC | % diff || PureJava | % diff || PureJavaC | % diff || Native | % diff || NativeC | % diff | | 32768 | 1 | 806.5 | 801.0 | -0.7% | 201.9 | -74.8% | 215.5 | 6.7% | 509.4 | 136.4% | 507.4 | -0.4% | | 32768 | 2 | 882.4 | 824.2 | -6.6% | 201.5 | -75.6% | 213.9 | 6.2% | 483.8 | 126.2% | 486.6 | 0.6% | | 32768 | 4 | 777.9 | 781.5 | 0.5% | 201.1 | -74.3% | 214.2 | 6.5% | 483.6 | 125.8% | 481.3 | -0.5% | | 32768 | 8 | 782.8 | 781.5 | -0.2% | 200.1 | -74.4% | 206.4 | 3.2% | 482.6 | 133.8% | 490.3 | 1.6% | | 32768 | 16 | 460.5 | 476.0 | 3.4% | 107.0 | -77.5% | 116.8 | 9.2% | 294.4 | 152.0% | 299.8 | 1.8% | | bpc | #T || Zip || ZipC | % diff || PureJava | % diff || PureJavaC | % diff || Native | % diff || NativeC | % diff | | 65536 | 1 | 801.1 | 806.1 | 0.6% | 202.1 | -74.9% | 215.4 | 6.6% | 509.1 | 136.3% | 509.6 | 0.1% | | 65536 | 2 | 782.8 | 887.4 | 13.4% | 201.0 | -77.3% | 214.7 | 6.8% | 482.9 | 125.0% | 485.0 | 0.4% | | 65536 | 4 | 775.1 | 786.4 | 1.4% | 205.9 | -73.8% | 208.1 | 1.1% | 662.3 | 218.3% | 662.2 | -0.0% | | 65536 | 8 | 889.3 | 875.5 | -1.5% | 204.8 | -76.6% | 216.1 | 5.5% | 507.1 | 134.7% | 547.1 | 7.9% | | 65536 | 16 | 468.3 | 458.6 | -2.1% | 106.3 | -76.8% | 114.0 | 7.2% | 288.8 | 153.3% | 308.7 | 6.9% | Please note that current JDK implementation is better than even the one in native library for both checksums. > Use java.util.zip.CRC32C for Java 9 and above > --------------------------------------------- > > Key: HADOOP-15033 > URL: https://issues.apache.org/jira/browse/HADOOP-15033 > Project: Hadoop Common > Issue Type: Improvement > Components: performance, util > Affects Versions: 3.0.0 > Reporter: Dmitry Chuyko > > java.util.zip.CRC32C implementation is available since Java 9. > https://docs.oracle.com/javase/9/docs/api/java/util/zip/CRC32C.html > Platform specific assembler intrinsics make it more effective than any pure > Java implementation. > Hadoop is compiled against Java 8 but class constructor may be accessible > with method handle on 9 to instances implementing Checksum in runtime. -- This message was sent by Atlassian JIRA (v6.4.14#64029) --------------------------------------------------------------------- To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org