[44/44] hadoop git commit: Merge branch 'trunk' into HDFS-7240

2018-04-13 Thread xyao 
Merge branch 'trunk' into HDFS-7240


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/72a3743c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/72a3743c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/72a3743c

Branch: refs/heads/HDFS-7240
Commit: 72a3743cc49d9c7b8d2eaec8064a25b8d890c267
Parents: 66610b5 995cba6
Author: Xiaoyu Yao 
Authored: Fri Apr 13 17:00:19 2018 -0700
Committer: Xiaoyu Yao 
Committed: Fri Apr 13 17:00:19 2018 -0700

--
 .../org/apache/hadoop/conf/Configuration.java   |  11 +-
 .../crypto/key/kms/KMSClientProvider.java   | 212 
 .../crypto/key/kms/KMSDelegationToken.java  |  22 +-
 .../crypto/key/kms/KMSLegacyTokenRenewer.java   |  56 ++
 .../hadoop/crypto/key/kms/KMSTokenRenewer.java  | 103 
 .../hadoop/crypto/key/kms/package-info.java |  18 +
 .../apache/hadoop/fs/ChecksumFileSystem.java|   9 +-
 .../fs/CommonConfigurationKeysPublic.java   |  10 +
 .../hadoop/fs/CompositeCrcFileChecksum.java |  82 +++
 .../java/org/apache/hadoop/fs/FileSystem.java   |   2 +-
 .../main/java/org/apache/hadoop/fs/Options.java |  11 +
 .../org/apache/hadoop/fs/shell/Command.java |  69 ++-
 .../apache/hadoop/fs/shell/CopyCommands.java|   6 +
 .../java/org/apache/hadoop/fs/shell/Ls.java |  26 +-
 .../org/apache/hadoop/fs/shell/PathData.java|  27 +
 .../web/DelegationTokenAuthenticatedURL.java|  21 +-
 .../DelegationTokenAuthenticationHandler.java   |   8 +-
 .../web/DelegationTokenAuthenticator.java   |   2 +-
 .../hadoop/service/launcher/IrqHandler.java |   2 +-
 .../org/apache/hadoop/util/CrcComposer.java | 187 +++
 .../java/org/apache/hadoop/util/CrcUtil.java| 220 
 .../org/apache/hadoop/util/DataChecksum.java|  18 +
 .../java/org/apache/hadoop/util/KMSUtil.java|  45 +-
 .../hadoop/util/KMSUtilFaultInjector.java   |  49 ++
 ...apache.hadoop.security.token.TokenIdentifier |   1 +
 ...rg.apache.hadoop.security.token.TokenRenewer |   3 +-
 .../src/main/resources/core-default.xml |  20 +
 .../apache/hadoop/conf/TestConfiguration.java   |  26 +-
 .../crypto/key/kms/TestKMSClientProvider.java   | 162 ++
 .../kms/TestLoadBalancingKMSClientProvider.java |  67 ++-
 .../apache/hadoop/fs/shell/find/TestFind.java   |  34 +-
 .../org/apache/hadoop/util/TestCrcComposer.java | 242 +
 .../org/apache/hadoop/util/TestCrcUtil.java | 232 +
 .../org/apache/hadoop/util/TestKMSUtil.java |  65 +++
 .../hadoop/crypto/key/kms/server/TestKMS.java   | 519 ---
 .../main/java/org/apache/hadoop/fs/Hdfs.java|   4 +-
 .../java/org/apache/hadoop/hdfs/DFSClient.java  |  56 +-
 .../hadoop/hdfs/DistributedFileSystem.java  |   5 +-
 .../apache/hadoop/hdfs/FileChecksumHelper.java  | 365 -
 .../hdfs/client/HdfsClientConfigKeys.java   |   2 +
 .../hadoop/hdfs/client/impl/DfsClientConf.java  |  27 +
 .../hdfs/protocol/BlockChecksumOptions.java |  54 ++
 .../hadoop/hdfs/protocol/BlockChecksumType.java |  30 ++
 .../datatransfer/DataTransferProtocol.java  |  12 +-
 .../hdfs/protocol/datatransfer/Sender.java  |  11 +-
 .../hadoop/hdfs/protocolPB/PBHelperClient.java  |  44 ++
 .../ha/RequestHedgingProxyProvider.java |   3 -
 .../hdfs/shortcircuit/ShortCircuitCache.java|  11 +-
 .../src/main/proto/datatransfer.proto   |   7 +-
 .../src/main/proto/hdfs.proto   |  21 +
 .../ha/TestRequestHedgingProxyProvider.java |  34 --
 .../native/libhdfspp/include/hdfspp/hdfspp.h|  53 +-
 .../native/libhdfspp/include/hdfspp/ioservice.h | 140 +
 .../native/libhdfspp/lib/bindings/c/hdfs.cc |   7 +-
 .../native/libhdfspp/lib/common/CMakeLists.txt  |   2 +-
 .../native/libhdfspp/lib/common/async_stream.h  |  13 +-
 .../libhdfspp/lib/common/continuation/asio.h|   5 -
 .../libhdfspp/lib/common/hdfs_ioservice.cc  | 146 --
 .../libhdfspp/lib/common/hdfs_ioservice.h   |  79 ---
 .../libhdfspp/lib/common/ioservice_impl.cc  | 159 ++
 .../libhdfspp/lib/common/ioservice_impl.h   |  76 +++
 .../main/native/libhdfspp/lib/common/logging.h  |   3 -
 .../libhdfspp/lib/common/namenode_info.cc   |  15 +-
 .../native/libhdfspp/lib/common/namenode_info.h |   8 +-
 .../main/native/libhdfspp/lib/common/util.cc|  14 +-
 .../src/main/native/libhdfspp/lib/common/util.h |  25 +-
 .../lib/connection/datanodeconnection.cc|  27 +-
 .../lib/connection/datanodeconnection.h |  26 +-
 .../main/native/libhdfspp/lib/fs/filehandle.cc  |  18 +-
 .../main/native/libhdfspp/lib/fs/filehandle.h   |  12 +-
 .../main/native/libhdfspp/lib/fs/filesystem.cc  |  67 ++-
 .../main/native/libhdfspp/lib/fs/filesystem.h   |  66 +--
 .../libhdfspp/lib/fs/namenode_operations.h  |   4 +-
 .../native/libhdfspp/lib/reader/block_reader.cc |  18 +-

[44/44] hadoop git commit: Merge branch 'trunk' into HDFS-7240

2017-11-01 Thread xyao 
Merge branch 'trunk' into HDFS-7240


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6e74039a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6e74039a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6e74039a

Branch: refs/heads/HDFS-7240
Commit: 6e74039a1a0fa7334946c8f15ce5cbcc9bbe17db
Parents: 919ae74 d57dba9
Author: Xiaoyu Yao 
Authored: Wed Nov 1 15:00:59 2017 -0700
Committer: Xiaoyu Yao 
Committed: Wed Nov 1 15:00:59 2017 -0700

--
 .../java/org/apache/hadoop/fs/FileSystem.java   |   46 +
 .../org/apache/hadoop/fs/FilterFileSystem.java  |   12 +
 .../org/apache/hadoop/fs/HarFileSystem.java |   14 +
 .../main/java/org/apache/hadoop/fs/Options.java |  180 ++
 .../java/org/apache/hadoop/fs/PathHandle.java   |   50 +
 .../org/apache/hadoop/fs/RawPathHandle.java |  119 +
 .../org/apache/hadoop/fs/shell/FsCommand.java   |1 +
 .../java/org/apache/hadoop/fs/shell/Head.java   |   78 +
 .../apache/hadoop/io/compress/BZip2Codec.java   |   39 +-
 .../io/compress/bzip2/CBZip2InputStream.java|   32 +-
 .../src/site/markdown/FileSystemShell.md|   13 +
 .../src/site/markdown/filesystem/filesystem.md  |  115 +
 .../fs/contract/AbstractContractOpenTest.java   |  247 +-
 .../hadoop/fs/contract/ContractOptions.java |   10 +
 .../hadoop/fs/contract/ContractTestUtils.java   |   32 +
 .../org/apache/hadoop/hdfs/DFSUtilClient.java   |   21 +
 .../hadoop/hdfs/DistributedFileSystem.java  |   54 +
 .../hdfs/client/impl/BlockReaderFactory.java|3 +-
 .../hadoop/hdfs/protocol/HdfsConstants.java |4 +
 .../hadoop/hdfs/protocol/HdfsFileStatus.java|  251 +-
 .../hadoop/hdfs/protocol/HdfsPathHandle.java|   98 +
 .../protocol/SnapshottableDirectoryStatus.java  |   16 +-
 .../hadoop/hdfs/protocolPB/PBHelperClient.java  |   17 +-
 .../apache/hadoop/hdfs/web/JsonUtilClient.java  |   29 +-
 .../src/main/proto/hdfs.proto   |7 +
 .../src/main/native/libhdfs/hdfs.c  |2 +-
 .../org/apache/hadoop/hdfs/DFSConfigKeys.java   |4 +
 .../server/datanode/DataNodeFaultInjector.java  |6 +
 .../hdfs/server/datanode/DataXceiver.java   |2 +
 .../erasurecode/ErasureCodingWorker.java|   12 +-
 .../erasurecode/StripedBlockReconstructor.java  |2 +
 .../federation/router/RouterRpcServer.java  |   18 +-
 .../server/namenode/FSDirStatAndListingOp.java  |   22 +-
 .../hdfs/server/namenode/FSDirectory.java   |   52 +-
 .../hdfs/server/namenode/FSNamesystem.java  |3 -
 .../src/main/resources/hdfs-default.xml |   13 +
 .../src/site/markdown/HDFSErasureCoding.md  |6 +
 .../hadoop/hdfs/TestDFSClientRetries.java   |   36 +-
 .../org/apache/hadoop/hdfs/TestDFSShell.java|   27 +
 .../org/apache/hadoop/hdfs/TestDFSUtil.java |   16 +
 .../apache/hadoop/hdfs/TestEncryptionZones.java |   30 +-
 .../hdfs/TestFileStatusSerialization.java   |   78 +-
 .../java/org/apache/hadoop/hdfs/TestLease.java  |   40 +-
 .../hadoop/hdfs/TestReconstructStripedFile.java |   64 +
 .../store/records/TestMountTable.java   |3 +-
 .../hadoop/hdfs/server/namenode/TestFsck.java   |   35 +-
 .../apache/hadoop/hdfs/web/TestJsonUtil.java|   19 +-
 .../src/test/resources/contract/hdfs.xml|   12 +-
 .../v2/app/job/impl/TaskAttemptImpl.java|  141 +-
 .../mapreduce/TestMapreduceConfigFields.java|   11 +
 .../v2/app/job/impl/TestTaskAttempt.java|  353 ++-
 .../apache/hadoop/mapreduce/MRJobConfig.java|   68 +-
 .../org/apache/hadoop/mapred/YARNRunner.java|   86 +-
 .../hadoop/mapred/TestTextInputFormat.java  |   76 +
 .../apache/hadoop/mapred/TestYARNRunner.java|  161 ++
 .../apache/hadoop/mapred/ShuffleHandler.java|   12 +-
 .../hadoop/mapred/TestShuffleHandler.java   |   75 +-
 hadoop-project/pom.xml  |   28 +-
 hadoop-tools/hadoop-resourceestimator/pom.xml   |2 +-
 .../resources/images/resourceestimator_arch.png |  Bin 0 -> 41981 bytes
 .../src/site/resources/images/tpch_history.png  |  Bin 0 -> 53335 bytes
 .../src/site/resources/images/tpch_predict.png  |  Bin 0 -> 56581 bytes
 .../org/apache/hadoop/yarn/sls/SLSRunner.java   |   85 +-
 .../hadoop/yarn/sls/conf/SLSConfiguration.java  |   34 +-
 .../apache/hadoop/yarn/sls/utils/SLSUtils.java  |   16 +-
 .../src/site/markdown/SchedulerLoadSimulator.md |2 +-
 .../hadoop-yarn/conf/container-executor.cfg |1 +
 .../Apache_Hadoop_YARN_Client_2.8.2.xml | 2316 ---
 .../Apache_Hadoop_YARN_Common_2.8.2.xml | 2665 --
 .../Apache_Hadoop_YARN_Server_Common_2.8.2.xml  |  829 --
 .../jdiff/Apache_Hadoop_YARN_Client_2.8.2.xml   | 2316 +++
 .../jdiff/Apache_Hadoop_YARN_Common_2.8.2.xml   | 2665 ++