Merge branch 'trunk' into HDFS-7240
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6e74039a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6e74039a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6e74039a Branch: refs/heads/HDFS-7240 Commit: 6e74039a1a0fa7334946c8f15ce5cbcc9bbe17db Parents: 919ae74 d57dba9 Author: Xiaoyu Yao <x...@apache.org> Authored: Wed Nov 1 15:00:59 2017 -0700 Committer: Xiaoyu Yao <x...@apache.org> Committed: Wed Nov 1 15:00:59 2017 -0700 ---------------------------------------------------------------------- .../java/org/apache/hadoop/fs/FileSystem.java | 46 + .../org/apache/hadoop/fs/FilterFileSystem.java | 12 + .../org/apache/hadoop/fs/HarFileSystem.java | 14 + .../main/java/org/apache/hadoop/fs/Options.java | 180 ++ .../java/org/apache/hadoop/fs/PathHandle.java | 50 + .../org/apache/hadoop/fs/RawPathHandle.java | 119 + .../org/apache/hadoop/fs/shell/FsCommand.java | 1 + .../java/org/apache/hadoop/fs/shell/Head.java | 78 + .../apache/hadoop/io/compress/BZip2Codec.java | 39 +- .../io/compress/bzip2/CBZip2InputStream.java | 32 +- .../src/site/markdown/FileSystemShell.md | 13 + .../src/site/markdown/filesystem/filesystem.md | 115 + .../fs/contract/AbstractContractOpenTest.java | 247 +- .../hadoop/fs/contract/ContractOptions.java | 10 + .../hadoop/fs/contract/ContractTestUtils.java | 32 + .../org/apache/hadoop/hdfs/DFSUtilClient.java | 21 + .../hadoop/hdfs/DistributedFileSystem.java | 54 + .../hdfs/client/impl/BlockReaderFactory.java | 3 +- .../hadoop/hdfs/protocol/HdfsConstants.java | 4 + .../hadoop/hdfs/protocol/HdfsFileStatus.java | 251 +- .../hadoop/hdfs/protocol/HdfsPathHandle.java | 98 + .../protocol/SnapshottableDirectoryStatus.java | 16 +- .../hadoop/hdfs/protocolPB/PBHelperClient.java | 17 +- .../apache/hadoop/hdfs/web/JsonUtilClient.java | 29 +- .../src/main/proto/hdfs.proto | 7 + .../src/main/native/libhdfs/hdfs.c | 2 +- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 + .../server/datanode/DataNodeFaultInjector.java | 6 + .../hdfs/server/datanode/DataXceiver.java | 2 + .../erasurecode/ErasureCodingWorker.java | 12 +- .../erasurecode/StripedBlockReconstructor.java | 2 + .../federation/router/RouterRpcServer.java | 18 +- .../server/namenode/FSDirStatAndListingOp.java | 22 +- .../hdfs/server/namenode/FSDirectory.java | 52 +- .../hdfs/server/namenode/FSNamesystem.java | 3 - .../src/main/resources/hdfs-default.xml | 13 + .../src/site/markdown/HDFSErasureCoding.md | 6 + .../hadoop/hdfs/TestDFSClientRetries.java | 36 +- .../org/apache/hadoop/hdfs/TestDFSShell.java | 27 + .../org/apache/hadoop/hdfs/TestDFSUtil.java | 16 + .../apache/hadoop/hdfs/TestEncryptionZones.java | 30 +- .../hdfs/TestFileStatusSerialization.java | 78 +- .../java/org/apache/hadoop/hdfs/TestLease.java | 40 +- .../hadoop/hdfs/TestReconstructStripedFile.java | 64 + .../store/records/TestMountTable.java | 3 +- .../hadoop/hdfs/server/namenode/TestFsck.java | 35 +- .../apache/hadoop/hdfs/web/TestJsonUtil.java | 19 +- .../src/test/resources/contract/hdfs.xml | 12 +- .../v2/app/job/impl/TaskAttemptImpl.java | 141 +- .../mapreduce/TestMapreduceConfigFields.java | 11 + .../v2/app/job/impl/TestTaskAttempt.java | 353 ++- .../apache/hadoop/mapreduce/MRJobConfig.java | 68 +- .../org/apache/hadoop/mapred/YARNRunner.java | 86 +- .../hadoop/mapred/TestTextInputFormat.java | 76 + .../apache/hadoop/mapred/TestYARNRunner.java | 161 ++ .../apache/hadoop/mapred/ShuffleHandler.java | 12 +- .../hadoop/mapred/TestShuffleHandler.java | 75 +- hadoop-project/pom.xml | 28 +- hadoop-tools/hadoop-resourceestimator/pom.xml | 2 +- .../resources/images/resourceestimator_arch.png | Bin 0 -> 41981 bytes .../src/site/resources/images/tpch_history.png | Bin 0 -> 53335 bytes .../src/site/resources/images/tpch_predict.png | Bin 0 -> 56581 bytes .../org/apache/hadoop/yarn/sls/SLSRunner.java | 85 +- .../hadoop/yarn/sls/conf/SLSConfiguration.java | 34 +- .../apache/hadoop/yarn/sls/utils/SLSUtils.java | 16 +- .../src/site/markdown/SchedulerLoadSimulator.md | 2 +- .../hadoop-yarn/conf/container-executor.cfg | 1 + .../Apache_Hadoop_YARN_Client_2.8.2.xml | 2316 --------------- .../Apache_Hadoop_YARN_Common_2.8.2.xml | 2665 ------------------ .../Apache_Hadoop_YARN_Server_Common_2.8.2.xml | 829 ------ .../jdiff/Apache_Hadoop_YARN_Client_2.8.2.xml | 2316 +++++++++++++++ .../jdiff/Apache_Hadoop_YARN_Common_2.8.2.xml | 2665 ++++++++++++++++++ .../Apache_Hadoop_YARN_Server_Common_2.8.2.xml | 829 ++++++ .../hadoop/yarn/api/records/Resource.java | 11 +- .../yarn/api/records/ResourceInformation.java | 1 - .../api/records/impl/LightWeightResource.java | 5 +- .../hadoop/yarn/conf/YarnConfiguration.java | 29 + .../server/api/AuxiliaryLocalPathHandler.java | 58 + .../yarn/server/api/AuxiliaryService.java | 21 + .../yarn/util/resource/ResourceUtils.java | 44 + .../distributedshell/ApplicationMaster.java | 13 +- .../applications/distributedshell/Client.java | 9 +- .../ContainerLaunchFailAppMaster.java | 10 +- .../distributedshell/TestDSFailedAppMaster.java | 7 +- .../TestDSSleepingAppMaster.java | 8 +- .../distributedshell/TestDistributedShell.java | 14 +- .../TestDistributedShellWithNodeLabels.java | 8 +- .../UnmanagedAMLauncher.java | 9 +- .../TestUnmanagedAMLauncher.java | 12 +- .../hadoop/yarn/client/api/AMRMClient.java | 7 +- .../yarn/client/api/async/AMRMClientAsync.java | 7 +- .../api/async/impl/AMRMClientAsyncImpl.java | 7 +- .../api/async/impl/NMClientAsyncImpl.java | 7 +- .../yarn/client/api/impl/AMRMClientImpl.java | 7 +- .../impl/ContainerManagementProtocolProxy.java | 7 +- .../yarn/client/api/impl/NMClientImpl.java | 7 +- .../client/api/impl/RemoteRequestsTable.java | 7 +- .../client/api/impl/SharedCacheClientImpl.java | 8 +- .../yarn/client/api/impl/YarnClientImpl.java | 7 +- .../apache/hadoop/yarn/client/cli/TopCLI.java | 7 +- .../hadoop/yarn/client/TestGetGroups.java | 7 +- .../hadoop/yarn/client/TestRMFailover.java | 10 +- ...nagerAdministrationProtocolPBClientImpl.java | 8 +- .../api/async/impl/TestAMRMClientAsync.java | 7 +- .../api/impl/AMRMTokenIdentifierForTest.java | 7 +- .../yarn/client/api/impl/TestAMRMProxy.java | 7 +- .../yarn/client/api/impl/TestNMClient.java | 8 +- .../api/impl/TestSharedCacheClientImpl.java | 8 +- .../yarn/client/api/impl/TestYarnClient.java | 10 +- .../ifile/IndexedFileAggregatedLogsBlock.java | 4 +- .../src/main/resources/yarn-default.xml | 42 +- .../impl/FSRegistryOperationsService.java | 249 ++ .../registry/client/types/ServiceRecord.java | 64 + .../impl/TestFSRegistryOperationsService.java | 298 ++ .../nodemanager/LinuxContainerExecutor.java | 3 +- .../containermanager/AuxServices.java | 6 +- .../containermanager/ContainerManagerImpl.java | 35 +- .../resources/gpu/GpuResourceAllocator.java | 102 +- .../resources/gpu/GpuResourceHandlerImpl.java | 90 +- .../runtime/DefaultLinuxContainerRuntime.java | 3 +- .../DelegatingLinuxContainerRuntime.java | 9 +- .../runtime/DockerLinuxContainerRuntime.java | 91 +- .../JavaSandboxLinuxContainerRuntime.java | 5 +- .../linux/runtime/LinuxContainerRuntime.java | 4 +- .../linux/runtime/docker/DockerRunCommand.java | 5 + .../runtime/docker/DockerVolumeCommand.java | 49 + .../resourceplugin/DockerCommandPlugin.java | 59 + .../resourceplugin/ResourcePlugin.java | 11 + .../resourceplugin/gpu/GpuDevice.java | 78 + .../resourceplugin/gpu/GpuDiscoverer.java | 30 +- .../gpu/GpuDockerCommandPluginFactory.java | 41 + .../gpu/GpuNodeResourceUpdateHandler.java | 10 +- .../resourceplugin/gpu/GpuResourcePlugin.java | 9 + .../gpu/NvidiaDockerV1CommandPlugin.java | 319 +++ .../recovery/NMLeveldbStateStoreService.java | 62 +- .../recovery/NMNullStateStoreService.java | 3 +- .../recovery/NMStateStoreService.java | 15 +- .../container-executor/impl/utils/docker-util.c | 130 + .../container-executor/impl/utils/docker-util.h | 18 +- .../test/utils/test_docker_util.cc | 42 + .../TestLinuxContainerExecutorWithMocks.java | 6 +- .../containermanager/TestAuxServices.java | 26 +- .../containermanager/TestContainerManager.java | 40 + .../TestContainerManagerRecovery.java | 9 +- .../resources/gpu/TestGpuResourceHandler.java | 156 +- .../TestDelegatingLinuxContainerRuntime.java | 14 +- .../runtime/TestDockerContainerRuntime.java | 204 +- .../TestJavaSandboxLinuxContainerRuntime.java | 3 +- .../docker/TestDockerCommandExecutor.java | 3 +- .../runtime/docker/TestDockerVolumeCommand.java | 45 + .../resourceplugin/gpu/TestGpuDiscoverer.java | 34 +- .../gpu/TestNvidiaDockerV1CommandPlugin.java | 217 ++ .../recovery/NMMemoryStateStoreService.java | 8 +- .../TestNMLeveldbStateStoreService.java | 22 +- .../scheduler/TestDistributedScheduler.java | 52 +- .../scheduler/fair/FairScheduler.java | 21 +- .../DominantResourceFairnessPolicy.java | 238 +- .../rmapp/TestApplicationLifetimeMonitor.java | 61 +- .../scheduler/fair/TestFSAppStarvation.java | 14 +- .../TestDominantResourceFairnessPolicy.java | 102 +- .../hadoop-yarn-server-router/pom.xml | 6 + .../hadoop/yarn/server/router/Router.java | 6 + .../webapp/DefaultRequestInterceptorREST.java | 12 +- .../webapp/FederationInterceptorREST.java | 255 +- .../router/webapp/RouterWebServiceUtil.java | 59 +- .../server/router/webapp/RouterWebServices.java | 97 +- .../webapp/BaseRouterWebServicesTest.java | 400 +-- .../yarn/server/router/webapp/JavaProcess.java | 15 +- .../webapp/TestRouterWebServicesREST.java | 1169 ++++---- .../src/site/markdown/CapacityScheduler.md | 52 + pom.xml | 2 +- 171 files changed, 12800 insertions(+), 7604 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e74039a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e74039a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e74039a/hadoop-project/pom.xml ---------------------------------------------------------------------- --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
