Merge branch 'trunk' into HDFS-7240
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/72a3743c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/72a3743c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/72a3743c Branch: refs/heads/HDFS-7240 Commit: 72a3743cc49d9c7b8d2eaec8064a25b8d890c267 Parents: 66610b5 995cba6 Author: Xiaoyu Yao <x...@apache.org> Authored: Fri Apr 13 17:00:19 2018 -0700 Committer: Xiaoyu Yao <x...@apache.org> Committed: Fri Apr 13 17:00:19 2018 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/conf/Configuration.java | 11 +- .../crypto/key/kms/KMSClientProvider.java | 212 ++++---- .../crypto/key/kms/KMSDelegationToken.java | 22 +- .../crypto/key/kms/KMSLegacyTokenRenewer.java | 56 ++ .../hadoop/crypto/key/kms/KMSTokenRenewer.java | 103 ++++ .../hadoop/crypto/key/kms/package-info.java | 18 + .../apache/hadoop/fs/ChecksumFileSystem.java | 9 +- .../fs/CommonConfigurationKeysPublic.java | 10 + .../hadoop/fs/CompositeCrcFileChecksum.java | 82 +++ .../java/org/apache/hadoop/fs/FileSystem.java | 2 +- .../main/java/org/apache/hadoop/fs/Options.java | 11 + .../org/apache/hadoop/fs/shell/Command.java | 69 ++- .../apache/hadoop/fs/shell/CopyCommands.java | 6 + .../java/org/apache/hadoop/fs/shell/Ls.java | 26 +- .../org/apache/hadoop/fs/shell/PathData.java | 27 + .../web/DelegationTokenAuthenticatedURL.java | 21 +- .../DelegationTokenAuthenticationHandler.java | 8 +- .../web/DelegationTokenAuthenticator.java | 2 +- .../hadoop/service/launcher/IrqHandler.java | 2 +- .../org/apache/hadoop/util/CrcComposer.java | 187 +++++++ .../java/org/apache/hadoop/util/CrcUtil.java | 220 ++++++++ .../org/apache/hadoop/util/DataChecksum.java | 18 + .../java/org/apache/hadoop/util/KMSUtil.java | 45 +- .../hadoop/util/KMSUtilFaultInjector.java | 49 ++ ...apache.hadoop.security.token.TokenIdentifier | 1 + ...rg.apache.hadoop.security.token.TokenRenewer | 3 +- .../src/main/resources/core-default.xml | 20 + .../apache/hadoop/conf/TestConfiguration.java | 26 +- .../crypto/key/kms/TestKMSClientProvider.java | 162 ++++++ .../kms/TestLoadBalancingKMSClientProvider.java | 67 ++- .../apache/hadoop/fs/shell/find/TestFind.java | 34 +- .../org/apache/hadoop/util/TestCrcComposer.java | 242 +++++++++ .../org/apache/hadoop/util/TestCrcUtil.java | 232 +++++++++ .../org/apache/hadoop/util/TestKMSUtil.java | 65 +++ .../hadoop/crypto/key/kms/server/TestKMS.java | 519 ++++++++++++++++--- .../main/java/org/apache/hadoop/fs/Hdfs.java | 4 +- .../java/org/apache/hadoop/hdfs/DFSClient.java | 56 +- .../hadoop/hdfs/DistributedFileSystem.java | 5 +- .../apache/hadoop/hdfs/FileChecksumHelper.java | 365 ++++++++----- .../hdfs/client/HdfsClientConfigKeys.java | 2 + .../hadoop/hdfs/client/impl/DfsClientConf.java | 27 + .../hdfs/protocol/BlockChecksumOptions.java | 54 ++ .../hadoop/hdfs/protocol/BlockChecksumType.java | 30 ++ .../datatransfer/DataTransferProtocol.java | 12 +- .../hdfs/protocol/datatransfer/Sender.java | 11 +- .../hadoop/hdfs/protocolPB/PBHelperClient.java | 44 ++ .../ha/RequestHedgingProxyProvider.java | 3 - .../hdfs/shortcircuit/ShortCircuitCache.java | 11 +- .../src/main/proto/datatransfer.proto | 7 +- .../src/main/proto/hdfs.proto | 21 + .../ha/TestRequestHedgingProxyProvider.java | 34 -- .../native/libhdfspp/include/hdfspp/hdfspp.h | 53 +- .../native/libhdfspp/include/hdfspp/ioservice.h | 140 +++++ .../native/libhdfspp/lib/bindings/c/hdfs.cc | 7 +- .../native/libhdfspp/lib/common/CMakeLists.txt | 2 +- .../native/libhdfspp/lib/common/async_stream.h | 13 +- .../libhdfspp/lib/common/continuation/asio.h | 5 - .../libhdfspp/lib/common/hdfs_ioservice.cc | 146 ------ .../libhdfspp/lib/common/hdfs_ioservice.h | 79 --- .../libhdfspp/lib/common/ioservice_impl.cc | 159 ++++++ .../libhdfspp/lib/common/ioservice_impl.h | 76 +++ .../main/native/libhdfspp/lib/common/logging.h | 3 - .../libhdfspp/lib/common/namenode_info.cc | 15 +- .../native/libhdfspp/lib/common/namenode_info.h | 8 +- .../main/native/libhdfspp/lib/common/util.cc | 14 +- .../src/main/native/libhdfspp/lib/common/util.h | 25 +- .../lib/connection/datanodeconnection.cc | 27 +- .../lib/connection/datanodeconnection.h | 26 +- .../main/native/libhdfspp/lib/fs/filehandle.cc | 18 +- .../main/native/libhdfspp/lib/fs/filehandle.h | 12 +- .../main/native/libhdfspp/lib/fs/filesystem.cc | 67 ++- .../main/native/libhdfspp/lib/fs/filesystem.h | 66 +-- .../libhdfspp/lib/fs/namenode_operations.h | 4 +- .../native/libhdfspp/lib/reader/block_reader.cc | 18 +- .../native/libhdfspp/lib/reader/block_reader.h | 10 +- .../native/libhdfspp/lib/reader/datatransfer.h | 4 +- .../libhdfspp/lib/rpc/namenode_tracker.cc | 2 +- .../native/libhdfspp/lib/rpc/namenode_tracker.h | 4 +- .../main/native/libhdfspp/lib/rpc/request.cc | 5 +- .../native/libhdfspp/lib/rpc/rpc_connection.h | 2 +- .../libhdfspp/lib/rpc/rpc_connection_impl.cc | 32 +- .../libhdfspp/lib/rpc/rpc_connection_impl.h | 9 +- .../main/native/libhdfspp/lib/rpc/rpc_engine.cc | 14 +- .../main/native/libhdfspp/lib/rpc/rpc_engine.h | 9 +- .../native/libhdfspp/tests/bad_datanode_test.cc | 31 +- .../libhdfspp/tests/hdfs_ioservice_test.cc | 10 +- .../native/libhdfspp/tests/mock_connection.h | 4 +- .../libhdfspp/tests/remote_block_reader_test.cc | 4 +- .../native/libhdfspp/tests/rpc_engine_test.cc | 112 ++-- .../resolver/FederationNamespaceInfo.java | 5 + .../federation/resolver/MountTableResolver.java | 4 +- .../federation/resolver/RemoteLocation.java | 35 +- .../router/RemoteLocationContext.java | 7 + .../federation/router/RouterRpcClient.java | 67 ++- .../federation/router/RouterRpcServer.java | 19 +- .../store/driver/impl/StateStoreFileImpl.java | 11 +- .../federation/store/records/MountTable.java | 2 +- .../store/records/impl/pb/MountTablePBImpl.java | 2 +- .../hdfs/server/federation/MockResolver.java | 12 +- .../federation/router/TestRouterAdmin.java | 4 +- .../federation/router/TestRouterMountTable.java | 62 ++- .../server/federation/router/TestRouterRpc.java | 48 ++ .../store/records/TestMountTable.java | 4 +- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 7 + .../hadoop/hdfs/net/DFSNetworkTopology.java | 10 +- .../hdfs/protocol/datatransfer/Receiver.java | 8 +- .../protocol/datatransfer/package-info.java | 8 +- .../common/blockaliasmap/package-info.java | 9 +- .../server/datanode/BlockChecksumHelper.java | 289 +++++++++-- .../hdfs/server/datanode/DataXceiver.java | 26 +- ...dBlockChecksumCompositeCrcReconstructor.java | 80 +++ ...StripedBlockChecksumMd5CrcReconstructor.java | 74 +++ .../StripedBlockChecksumReconstructor.java | 66 ++- .../erasurecode/StripedBlockReconstructor.java | 1 + .../impl/FsDatasetAsyncDiskService.java | 3 +- .../datanode/fsdataset/impl/FsVolumeList.java | 2 +- .../diskbalancer/connectors/package-info.java | 3 +- .../diskbalancer/datamodel/package-info.java | 4 +- .../hdfs/server/diskbalancer/package-info.java | 2 +- .../diskbalancer/planner/package-info.java | 2 +- .../org/apache/hadoop/hdfs/tools/DFSck.java | 2 +- .../src/main/resources/hdfs-default.xml | 25 + .../src/site/markdown/TransparentEncryption.md | 57 +- .../org/apache/hadoop/hdfs/DFSTestUtil.java | 31 +- .../apache/hadoop/hdfs/TestEncryptionZones.java | 3 + .../apache/hadoop/hdfs/TestFileChecksum.java | 101 +++- .../hdfs/TestFileChecksumCompositeCrc.java | 47 ++ .../hadoop/hdfs/protocolPB/TestPBHelper.java | 14 + .../blockmanagement/TestDatanodeManager.java | 52 ++ .../shortcircuit/TestShortCircuitCache.java | 26 + .../apache/hadoop/mapred/MapReduceChildJVM.java | 73 +-- .../v2/app/job/impl/TaskAttemptImpl.java | 8 +- .../v2/app/job/impl/TestMapReduceChildJVM.java | 24 +- .../apache/hadoop/mapreduce/v2/util/MRApps.java | 10 + .../java/org/apache/hadoop/mapred/JobConf.java | 18 + .../apache/hadoop/mapred/pipes/Application.java | 5 +- .../src/main/resources/mapred-default.xml | 67 ++- .../src/site/markdown/MapReduceTutorial.md | 6 + .../HistoryServerLeveldbStateStoreService.java | 11 - .../org/apache/hadoop/mapred/YARNRunner.java | 11 +- .../apache/hadoop/mapred/TestYARNRunner.java | 26 +- .../mapred/pipes/TestPipeApplication.java | 19 +- .../mapreduce/MiniHadoopClusterManager.java | 1 + .../apache/hadoop/mapred/ShuffleHandler.java | 12 - hadoop-maven-plugins/pom.xml | 2 +- .../apache/hadoop/fs/s3a/S3ARetryPolicy.java | 6 +- .../org/apache/hadoop/fs/s3a/TestInvoker.java | 14 +- .../hadoop/tools/mapred/TestCopyMapper.java | 173 +++++-- .../mapred/TestCopyMapperCompositeCrc.java | 50 ++ .../hadoop/yarn/api/ApplicationConstants.java | 10 + .../hadoop/yarn/service/webapp/ApiServer.java | 47 +- .../hadoop/yarn/service/ServiceClientTest.java | 6 + .../hadoop/yarn/service/TestApiServer.java | 26 + .../hadoop/yarn/service/ClientAMService.java | 1 + .../hadoop/yarn/service/ServiceScheduler.java | 41 +- .../yarn/service/client/ServiceClient.java | 102 +++- .../yarn/service/component/Component.java | 1 + .../yarn/service/conf/YarnServiceConf.java | 2 + .../hadoop/yarn/service/ServiceTestUtils.java | 26 +- .../yarn/service/TestYarnNativeServices.java | 129 ++++- .../java/org/apache/hadoop/yarn/util/Apps.java | 115 +++- .../org/apache/hadoop/yarn/util/TestApps.java | 136 +++++ .../server/nodemanager/ContainerExecutor.java | 11 + .../nodemanager/DefaultContainerExecutor.java | 6 + .../nodemanager/LinuxContainerExecutor.java | 124 +++-- .../container/ContainerImpl.java | 3 +- .../launcher/ContainerLaunch.java | 20 +- .../launcher/ContainerRelaunch.java | 2 +- .../runtime/DefaultLinuxContainerRuntime.java | 6 + .../DelegatingLinuxContainerRuntime.java | 9 + .../runtime/DockerLinuxContainerRuntime.java | 50 +- .../JavaSandboxLinuxContainerRuntime.java | 10 + .../runtime/docker/DockerCommandExecutor.java | 14 + .../runtime/docker/DockerStartCommand.java | 29 ++ .../runtime/ContainerRuntime.java | 10 + .../scheduler/ContainerScheduler.java | 7 +- .../recovery/NMLeveldbStateStoreService.java | 16 +- .../impl/container-executor.c | 2 +- .../container-executor/impl/utils/docker-util.c | 140 ++++- .../container-executor/impl/utils/docker-util.h | 11 + .../test/utils/test_docker_util.cc | 117 +++-- .../nodemanager/TestLinuxContainerExecutor.java | 12 + .../TestContainerManagerRecovery.java | 51 ++ .../launcher/TestContainerRelaunch.java | 2 +- .../runtime/TestDockerContainerRuntime.java | 101 +++- .../runtime/docker/TestDockerStartCommand.java | 53 ++ .../TestContainersMonitorResourceChange.java | 5 + .../TestNMLeveldbStateStoreService.java | 16 + .../recovery/LeveldbRMStateStore.java | 12 - .../scheduler/common/fica/FiCaSchedulerApp.java | 10 + .../resourcemanager/TestClientRMService.java | 14 +- .../TestCapacitySchedulerAsyncScheduling.java | 91 ++++ .../storage/TimelineSchemaCreator.java | 10 + .../storage/domain/DomainTableRW.java | 92 ++++ .../storage/domain/package-info.java | 28 + .../storage/domain/DomainColumn.java | 111 ++++ .../storage/domain/DomainColumnFamily.java | 52 ++ .../storage/domain/DomainRowKey.java | 179 +++++++ .../storage/domain/DomainTable.java | 45 ++ .../storage/domain/package-info.java | 28 + .../storage/common/TestRowKeys.java | 32 ++ .../site/markdown/PlacementConstraints.md.vm | 15 +- .../src/site/markdown/yarn-service/Concepts.md | 6 +- .../src/site/markdown/yarn-service/Overview.md | 14 +- .../site/markdown/yarn-service/QuickStart.md | 12 +- .../site/markdown/yarn-service/RegistryDNS.md | 4 +- .../markdown/yarn-service/ServiceDiscovery.md | 4 +- 207 files changed, 6996 insertions(+), 1510 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/72a3743c/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml ---------------------------------------------------------------------- --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
