Merge branch 'trunk' into s3guard/HADOOP-13345 # Conflicts: # hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0c61010d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0c61010d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0c61010d Branch: refs/heads/HADOOP-13345 Commit: 0c61010deec872a56149e58055fc90aba08fbb83 Parents: a5cc315 e648b6e Author: Steve Loughran <ste...@apache.org> Authored: Wed Jan 11 18:18:49 2017 +0000 Committer: Steve Loughran <ste...@apache.org> Committed: Wed Jan 11 18:18:49 2017 +0000 ---------------------------------------------------------------------- .gitignore | 1 + BUILDING.txt | 25 + LICENSE.txt | 1322 +++++++++++------- NOTICE.txt | 129 +- dev-support/bin/dist-copynativelibs | 13 + dev-support/bin/dist-layout-stitching | 8 +- .../resources/assemblies/hadoop-kms-dist.xml | 25 +- hadoop-client-modules/hadoop-client-api/pom.xml | 216 +++ .../hadoop-client-check-invariants/pom.xml | 124 ++ .../hadoop-client-check-test-invariants/pom.xml | 132 ++ .../hadoop-client-integration-tests/pom.xml | 130 ++ .../apache/hadoop/example/ITUseMiniCluster.java | 113 ++ .../src/test/resources/hdfs-site.xml | 34 + .../src/test/resources/log4j.properties | 24 + .../hadoop-client-minicluster/pom.xml | 745 ++++++++++ .../hadoop-client-runtime/pom.xml | 318 +++++ hadoop-client-modules/hadoop-client/pom.xml | 337 +++++ hadoop-client-modules/pom.xml | 45 + hadoop-client/pom.xml | 319 ----- .../server/AuthenticationFilter.java | 6 +- .../server/KerberosAuthenticationHandler.java | 254 +++- .../authentication/util/KerberosName.java | 4 +- .../authentication/util/TestKerberosName.java | 22 + hadoop-common-project/hadoop-common/pom.xml | 24 +- .../hadoop-common/src/CMakeLists.txt | 29 + .../hadoop-common/src/config.h.cmake | 1 + .../src/main/bin/hadoop-functions.sh | 51 +- .../hadoop/conf/ConfigurationWithLogging.java | 113 ++ .../hadoop/fs/CommonConfigurationKeys.java | 16 + .../main/java/org/apache/hadoop/fs/FsShell.java | 7 +- .../org/apache/hadoop/fs/ftp/FTPFileSystem.java | 70 +- .../hadoop/fs/viewfs/ChRootedFileSystem.java | 5 + .../apache/hadoop/fs/viewfs/ViewFileSystem.java | 11 + .../apache/hadoop/ha/ZKFailoverController.java | 7 +- .../org/apache/hadoop/http/HttpServer2.java | 163 ++- .../apache/hadoop/io/compress/Decompressor.java | 2 +- .../hadoop/io/compress/ZStandardCodec.java | 242 ++++ .../io/compress/zstd/ZStandardCompressor.java | 305 ++++ .../io/compress/zstd/ZStandardDecompressor.java | 323 +++++ .../hadoop/io/compress/zstd/package-info.java | 22 + .../org/apache/hadoop/io/nativeio/NativeIO.java | 40 +- .../hadoop/metrics2/MetricsJsonBuilder.java | 125 ++ .../hadoop/metrics2/lib/MutableQuantiles.java | 10 + .../lib/MutableRatesWithAggregation.java | 40 +- .../hadoop/metrics2/lib/RollingAverages.java | 251 ++++ .../apache/hadoop/security/ssl/SSLFactory.java | 64 +- .../hadoop/util/DiskValidatorFactory.java | 5 +- .../hadoop/util/GenericOptionsParser.java | 14 +- .../apache/hadoop/util/NativeCodeLoader.java | 5 + .../hadoop/util/NativeLibraryChecker.java | 13 +- .../hadoop/util/ReadWriteDiskValidator.java | 95 ++ .../util/ReadWriteDiskValidatorMetrics.java | 170 +++ .../main/java/org/apache/hadoop/util/Shell.java | 24 + .../io/compress/zstd/ZStandardCompressor.c | 259 ++++ .../io/compress/zstd/ZStandardDecompressor.c | 218 +++ .../zstd/org_apache_hadoop_io_compress_zstd.h | 34 + .../org/apache/hadoop/util/NativeCodeLoader.c | 11 + ...g.apache.hadoop.io.compress.CompressionCodec | 1 + .../src/main/resources/core-default.xml | 68 +- .../src/site/markdown/ClusterSetup.md | 1 + .../src/site/markdown/CommandsManual.md | 15 +- .../hadoop-common/src/site/markdown/Metrics.md | 37 + .../src/site/markdown/NativeLibraries.md.vm | 1 + .../src/site/markdown/SingleCluster.md.vm | 12 +- .../conf/TestCommonConfigurationFields.java | 11 +- .../java/org/apache/hadoop/fs/TestFsShell.java | 67 +- .../apache/hadoop/fs/ftp/TestFTPFileSystem.java | 56 +- .../fs/viewfs/ViewFileSystemBaseTest.java | 73 + .../apache/hadoop/io/compress/TestCodec.java | 12 + .../io/compress/TestCompressionStreamReuse.java | 8 + .../TestZStandardCompressorDecompressor.java | 485 +++++++ .../hadoop/metrics2/impl/MetricsRecords.java | 16 + .../metrics2/lib/TestRollingAverages.java | 123 ++ .../delegation/web/TestWebDelegationToken.java | 4 + .../apache/hadoop/test/GenericTestUtils.java | 72 +- .../org/apache/hadoop/test/MetricsAsserts.java | 7 + .../hadoop/util/TestReadWriteDiskValidator.java | 161 +++ .../java/org/apache/hadoop/util/TestShell.java | 52 + .../src/test/resources/zstd/test_file.txt | 71 + .../src/test/resources/zstd/test_file.txt.zst | Bin 0 -> 3690 bytes .../src/test/scripts/hadoop_mkdir.bats | 42 + .../src/test/scripts/hadoop_using_envvar.bats | 33 + .../dev-support/findbugsExcludeFile.xml | 2 +- hadoop-common-project/hadoop-kms/pom.xml | 160 +-- .../hadoop-kms/src/main/conf/kms-env.sh | 53 +- .../src/main/conf/kms-log4j.properties | 3 +- .../hadoop-kms/src/main/conf/kms-site.xml | 167 +-- .../hadoop/crypto/key/kms/server/KMSACLs.java | 36 +- .../hadoop/crypto/key/kms/server/KMSAudit.java | 30 +- .../crypto/key/kms/server/KMSAuditLogger.java | 21 +- .../crypto/key/kms/server/KMSConfiguration.java | 16 + .../crypto/key/kms/server/KMSJMXServlet.java | 36 - .../hadoop/crypto/key/kms/server/KMSWebApp.java | 23 +- .../crypto/key/kms/server/KMSWebServer.java | 155 ++ .../hadoop-kms/src/main/libexec/kms-config.sh | 77 - .../main/libexec/shellprofile.d/hadoop-kms.sh | 57 + .../src/main/resources/kms-default.xml | 248 ++++ .../main/resources/webapps/kms/WEB-INF/web.xml | 68 + .../main/resources/webapps/static/index.html | 35 + .../hadoop-kms/src/main/sbin/kms.sh | 123 +- .../src/main/tomcat/ROOT/WEB-INF/web.xml | 16 - .../hadoop-kms/src/main/tomcat/ROOT/index.html | 27 - .../src/main/tomcat/logging.properties | 67 - .../hadoop-kms/src/main/tomcat/server.xml | 155 -- .../src/main/tomcat/ssl-server.xml.conf | 137 -- .../hadoop-kms/src/main/webapp/WEB-INF/web.xml | 78 -- .../hadoop-kms/src/site/configuration.xsl | 49 + .../hadoop-kms/src/site/markdown/index.md.vm | 122 +- .../hadoop/crypto/key/kms/server/MiniKMS.java | 118 +- hadoop-dist/pom.xml | 17 + .../java/org/apache/hadoop/hdfs/DFSClient.java | 16 +- .../hadoop/hdfs/DistributedFileSystem.java | 29 + .../apache/hadoop/hdfs/client/HdfsAdmin.java | 10 + .../hadoop/hdfs/protocol/ClientProtocol.java | 7 + .../datatransfer/BlockPinningException.java | 33 + .../datatransfer/DataTransferProtoUtil.java | 17 +- .../ClientNamenodeProtocolTranslatorPB.java | 15 + .../server/datanode/BlockMetadataHeader.java | 29 +- .../server/datanode/DiskBalancerWorkItem.java | 2 +- .../server/datanode/DiskBalancerWorkStatus.java | 4 +- .../hdfs/util/CombinedHostsFileReader.java | 2 +- .../apache/hadoop/hdfs/web/JsonUtilClient.java | 55 +- .../hadoop/hdfs/web/WebHdfsFileSystem.java | 73 +- ...onfRefreshTokenBasedAccessTokenProvider.java | 2 +- .../CredentialBasedAccessTokenProvider.java | 2 +- .../hdfs/web/resources/CreateParentParam.java | 2 +- .../hadoop/hdfs/web/resources/GetOpParam.java | 12 +- .../hdfs/web/resources/OverwriteParam.java | 2 +- .../src/main/proto/ClientNamenodeProtocol.proto | 2 + .../src/main/proto/datatransfer.proto | 1 + .../src/main/proto/erasurecoding.proto | 7 + .../dev-support/findbugsExcludeFile.xml | 27 + .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 25 +- ...tNamenodeProtocolServerSideTranslatorPB.java | 14 + .../apache/hadoop/hdfs/protocolPB/PBHelper.java | 15 +- .../hadoop/hdfs/server/balancer/Dispatcher.java | 62 +- .../blockmanagement/DatanodeDescriptor.java | 17 +- .../server/blockmanagement/DatanodeManager.java | 8 +- .../blockmanagement/DecommissionManager.java | 9 +- .../server/blockmanagement/NumberReplicas.java | 2 +- .../hdfs/server/common/HdfsServerConstants.java | 9 +- .../server/common/HttpGetFailedException.java | 39 + .../server/common/HttpPutFailedException.java | 37 + .../hadoop/hdfs/server/common/Storage.java | 12 +- .../apache/hadoop/hdfs/server/common/Util.java | 258 +++- .../hdfs/server/datanode/BPServiceActor.java | 10 +- .../server/datanode/BlockPoolSliceStorage.java | 8 +- .../hdfs/server/datanode/BlockReceiver.java | 84 +- .../hdfs/server/datanode/BlockScanner.java | 10 +- .../hdfs/server/datanode/BlockSender.java | 30 +- .../hadoop/hdfs/server/datanode/DataNode.java | 207 +-- .../server/datanode/DataNodeFaultInjector.java | 25 +- .../hdfs/server/datanode/DataNodeMXBean.java | 18 +- .../hdfs/server/datanode/DataStorage.java | 16 +- .../hdfs/server/datanode/DataXceiver.java | 17 +- .../hdfs/server/datanode/DatanodeUtil.java | 19 +- .../datanode/FaultInjectorFileIoEvents.java | 55 + .../hdfs/server/datanode/FileIoProvider.java | 1051 ++++++++++++++ .../hdfs/server/datanode/FinalizedReplica.java | 29 + .../hdfs/server/datanode/LocalReplica.java | 133 +- .../server/datanode/LocalReplicaInPipeline.java | 41 +- .../server/datanode/ProfilingFileIoEvents.java | 124 ++ .../hdfs/server/datanode/ReplicaInPipeline.java | 4 +- .../hdfs/server/datanode/ReplicaInfo.java | 17 +- .../server/datanode/ReplicaUnderRecovery.java | 2 +- .../hdfs/server/datanode/VolumeScanner.java | 9 +- .../server/datanode/checker/AsyncChecker.java | 5 +- .../datanode/checker/DatasetVolumeChecker.java | 178 ++- .../checker/StorageLocationChecker.java | 8 +- .../datanode/checker/ThrottledAsyncChecker.java | 19 +- .../erasurecode/StripedBlockReader.java | 10 +- .../erasurecode/StripedBlockReconstructor.java | 7 +- .../erasurecode/StripedBlockWriter.java | 1 + .../datanode/erasurecode/StripedReader.java | 4 + .../erasurecode/StripedReconstructor.java | 31 + .../datanode/erasurecode/StripedWriter.java | 4 + .../fsdataset/DataNodeVolumeMetrics.java | 289 ++++ .../server/datanode/fsdataset/FsDatasetSpi.java | 3 +- .../server/datanode/fsdataset/FsVolumeSpi.java | 16 + .../datanode/fsdataset/ReplicaInputStreams.java | 11 +- .../fsdataset/ReplicaOutputStreams.java | 72 +- .../datanode/fsdataset/impl/BlockPoolSlice.java | 123 +- .../impl/FsDatasetAsyncDiskService.java | 6 +- .../datanode/fsdataset/impl/FsDatasetImpl.java | 53 +- .../datanode/fsdataset/impl/FsDatasetUtil.java | 5 +- .../datanode/fsdataset/impl/FsVolumeImpl.java | 133 +- .../fsdataset/impl/FsVolumeImplBuilder.java | 12 +- .../datanode/fsdataset/impl/FsVolumeList.java | 25 +- .../datanode/metrics/DataNodeMetrics.java | 24 +- .../datanode/metrics/DataNodePeerMetrics.java | 117 ++ .../server/datanode/web/webhdfs/HdfsWriter.java | 19 +- .../server/diskbalancer/command/Command.java | 2 +- .../connectors/JsonNodeConnector.java | 2 +- .../datamodel/DiskBalancerCluster.java | 2 +- .../datamodel/DiskBalancerVolume.java | 2 +- .../server/diskbalancer/planner/NodePlan.java | 2 +- .../apache/hadoop/hdfs/server/mover/Mover.java | 26 +- .../server/namenode/EditLogFileInputStream.java | 3 +- .../server/namenode/EncryptionZoneManager.java | 8 +- .../hadoop/hdfs/server/namenode/FSDirAclOp.java | 6 +- .../hdfs/server/namenode/FSDirAttrOp.java | 16 +- .../server/namenode/FSDirErasureCodingOp.java | 115 +- .../hadoop/hdfs/server/namenode/FSEditLog.java | 18 +- .../hadoop/hdfs/server/namenode/FSImage.java | 3 - .../server/namenode/FSImageFormatProtobuf.java | 4 +- ...FSImagePreTransactionalStorageInspector.java | 2 +- .../hdfs/server/namenode/FSNamesystem.java | 65 +- .../hdfs/server/namenode/ImageServlet.java | 15 +- .../hadoop/hdfs/server/namenode/NNStorage.java | 14 +- .../hadoop/hdfs/server/namenode/NameNode.java | 5 +- .../hdfs/server/namenode/NameNodeMXBean.java | 15 +- .../hdfs/server/namenode/NameNodeRpcServer.java | 20 +- .../server/namenode/NameNodeStatusMXBean.java | 8 +- .../hdfs/server/namenode/NamenodeFsck.java | 61 +- .../hdfs/server/namenode/TransferFsImage.java | 295 +--- .../server/namenode/ha/BootstrapStandby.java | 10 +- .../web/resources/NamenodeWebHdfsMethods.java | 16 + .../apache/hadoop/hdfs/tools/CacheAdmin.java | 8 +- .../apache/hadoop/hdfs/tools/CryptoAdmin.java | 8 +- .../org/apache/hadoop/hdfs/tools/DFSck.java | 4 + .../hadoop/hdfs/tools/StoragePolicyAdmin.java | 6 +- .../hdfs/tools/erasurecode/ECCommand.java | 34 + .../org/apache/hadoop/hdfs/web/JsonUtil.java | 33 + .../src/main/proto/DatanodeProtocol.proto | 10 - .../hadoop-hdfs/src/main/proto/HdfsServer.proto | 15 +- .../src/main/resources/hdfs-default.xml | 25 + .../src/main/webapps/hdfs/dfshealth.html | 36 +- .../src/main/webapps/hdfs/dfshealth.js | 11 +- .../src/site/markdown/HDFSCommands.md | 6 +- .../src/site/markdown/HDFSErasureCoding.md | 5 + .../src/site/markdown/HdfsRollingUpgrade.md | 8 +- .../hadoop-hdfs/src/site/markdown/LibHdfs.md | 2 +- .../hadoop-hdfs/src/site/markdown/WebHDFS.md | 197 ++- .../apache/hadoop/cli/TestCryptoAdminCLI.java | 3 +- .../org/apache/hadoop/hdfs/MiniDFSCluster.java | 15 +- .../apache/hadoop/hdfs/StripedFileTestUtil.java | 24 + .../TestClientProtocolForPipelineRecovery.java | 3 +- .../hadoop/hdfs/TestErasureCodingPolicies.java | 5 +- .../org/apache/hadoop/hdfs/TestFileAppend.java | 2 +- .../org/apache/hadoop/hdfs/TestSafeMode.java | 1 + .../TestUnsetAndChangeDirectoryEcPolicy.java | 366 +++++ .../blockmanagement/BlockManagerTestUtil.java | 4 +- .../blockmanagement/TestBlockStatsMXBean.java | 4 + .../hdfs/server/common/StorageAdapter.java | 2 +- .../hdfs/server/datanode/DataNodeTestUtils.java | 82 ++ .../server/datanode/SimulatedFSDataset.java | 56 +- .../server/datanode/TestBPOfferService.java | 1 + .../datanode/TestBlockPoolSliceStorage.java | 2 +- .../hdfs/server/datanode/TestBlockRecovery.java | 2 +- .../server/datanode/TestBlockReplacement.java | 70 +- .../hdfs/server/datanode/TestBlockScanner.java | 100 ++ .../TestDataNodeErasureCodingMetrics.java | 168 ++- .../datanode/TestDataNodeFaultInjector.java | 173 +++ .../datanode/TestDataNodeHotSwapVolumes.java | 177 ++- .../server/datanode/TestDataNodeLifeline.java | 33 +- .../datanode/TestDataNodePeerMetrics.java | 92 ++ .../datanode/TestDataNodeVolumeFailure.java | 20 +- .../TestDataNodeVolumeFailureReporting.java | 66 +- .../TestDataNodeVolumeFailureToleration.java | 3 + .../datanode/TestDataNodeVolumeMetrics.java | 182 +++ .../server/datanode/TestDirectoryScanner.java | 17 + .../hdfs/server/datanode/TestDiskError.java | 24 +- .../server/datanode/TestSimulatedFSDataset.java | 2 +- .../checker/TestDatasetVolumeChecker.java | 97 +- .../TestDatasetVolumeCheckerFailures.java | 60 +- .../checker/TestThrottledAsyncChecker.java | 128 +- .../extdataset/ExternalDatasetImpl.java | 5 +- .../extdataset/ExternalReplicaInPipeline.java | 6 +- .../datanode/extdataset/ExternalVolumeImpl.java | 19 + .../fsdataset/impl/TestFsDatasetImpl.java | 84 +- .../fsdataset/impl/TestFsVolumeList.java | 37 - .../hadoop/hdfs/server/mover/TestMover.java | 163 ++- .../namenode/TestEncryptionZoneManager.java | 138 ++ .../hdfs/server/namenode/TestFSDirAttrOp.java | 76 + .../hdfs/server/namenode/TestFSImage.java | 12 +- .../hdfs/server/namenode/TestFileTruncate.java | 31 +- .../hadoop/hdfs/server/namenode/TestFsck.java | 340 ++++- .../server/namenode/TestNameNodeMXBean.java | 225 ++- .../apache/hadoop/hdfs/web/TestJsonUtil.java | 8 +- .../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 315 +++++ .../hdfs/web/TestWebHdfsFileSystemContract.java | 40 + .../hadoop/tools/TestHdfsConfigFields.java | 4 + .../src/test/resources/testCryptoConf.xml | 98 +- .../test/resources/testErasureCodingConf.xml | 64 +- .../jobhistory/JobHistoryCopyService.java | 5 +- .../hadoop/mapreduce/v2/app/AppContext.java | 4 +- .../hadoop/mapreduce/v2/app/MRAppMaster.java | 2 +- .../mapreduce/v2/app/job/impl/JobImpl.java | 4 +- .../v2/app/rm/RMContainerAllocator.java | 20 +- .../v2/app/speculate/DefaultSpeculator.java | 3 +- .../mapred/TestLocalContainerLauncher.java | 2 +- .../mapred/TestTaskAttemptFinishingMonitor.java | 2 +- .../mapred/TestTaskAttemptListenerImpl.java | 17 +- .../jobhistory/TestJobHistoryEventHandler.java | 10 +- .../hadoop/mapreduce/v2/app/MockAppContext.java | 4 +- .../mapreduce/v2/app/MockEventHandler.java | 6 +- .../v2/app/TestCheckpointPreemptionPolicy.java | 4 +- .../mapreduce/v2/app/TestFetchFailure.java | 17 +- .../v2/app/TestKillAMPreemptionPolicy.java | 4 +- .../mapreduce/v2/app/TestRuntimeEstimators.java | 3 +- .../app/commit/TestCommitterEventHandler.java | 2 +- .../mapreduce/v2/app/job/impl/TestJobImpl.java | 8 + .../app/launcher/TestContainerLauncherImpl.java | 12 +- .../app/local/TestLocalContainerAllocator.java | 4 +- .../v2/app/rm/TestRMContainerAllocator.java | 12 +- .../apache/hadoop/mapreduce/TypeConverter.java | 7 +- .../org/apache/hadoop/mapred/CleanupQueue.java | 7 +- .../java/org/apache/hadoop/mapred/JobConf.java | 2 +- .../java/org/apache/hadoop/mapred/MapTask.java | 10 +- .../java/org/apache/hadoop/mapred/Mapper.java | 2 +- .../java/org/apache/hadoop/mapred/Reducer.java | 2 +- .../org/apache/hadoop/mapreduce/JobStatus.java | 18 +- .../apache/hadoop/mapreduce/JobSubmitter.java | 2 +- .../apache/hadoop/mapreduce/MRJobConfig.java | 6 +- .../mapreduce/lib/output/TextOutputFormat.java | 15 +- .../task/reduce/ShuffleSchedulerImpl.java | 40 +- .../src/site/markdown/MapredCommands.md | 6 + .../hadoop/mapreduce/v2/hs/JobHistory.java | 3 +- .../resources/job_1329348432655_0001_conf.xml | 2 - .../apache/hadoop/mapred/JobClientUnitTest.java | 6 +- .../hadoop/mapred/TestMROpportunisticMaps.java | 2 +- .../mapred/pipes/TestPipeApplication.java | 4 +- .../hadoop-mapreduce-client-nativetask/pom.xml | 8 + .../hadoop/examples/dancing/Pentomino.java | 5 + .../hadoop/examples/terasort/TeraScheduler.java | 16 +- hadoop-maven-plugins/pom.xml | 22 + .../resource/ServicesResourceTransformer.java | 164 +++ .../plugin/shade/resource/package-info.java | 23 + hadoop-project-dist/pom.xml | 8 +- hadoop-project/pom.xml | 130 +- .../hadoop/tools/TestHadoopArchiveLogs.java | 12 +- .../tools/TestHadoopArchiveLogsRunner.java | 11 +- .../org/apache/hadoop/fs/s3a/Constants.java | 14 + .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 16 +- .../apache/hadoop/fs/s3a/S3AInputStream.java | 33 +- .../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 100 +- .../src/site/markdown/tools/hadoop-aws/index.md | 153 +- .../tools/hadoop-aws/troubleshooting_s3a.md | 52 + .../fs/s3a/ITestS3AAWSCredentialsProvider.java | 1 - .../hadoop/fs/s3a/ITestS3AConfiguration.java | 130 ++ .../apache/hadoop/fs/s3a/S3ATestConstants.java | 12 - .../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 61 +- .../scale/ITestS3AInputStreamPerformance.java | 57 +- .../hadoop/fs/s3a/scale/S3AScaleTestBase.java | 28 +- hadoop-tools/hadoop-azure-datalake/pom.xml | 10 +- .../fs/azure/AzureNativeFileSystemStore.java | 91 +- .../org/apache/hadoop/fs/azure/KeyProvider.java | 3 +- .../fs/azure/LocalSASKeyGeneratorImpl.java | 263 ++++ .../hadoop/fs/azure/NativeAzureFileSystem.java | 31 +- .../hadoop/fs/azure/NativeFileSystemStore.java | 5 +- .../fs/azure/RemoteSASKeyGeneratorImpl.java | 296 ++++ .../fs/azure/SASKeyGenerationException.java | 40 + .../hadoop/fs/azure/SASKeyGeneratorImpl.java | 61 + .../fs/azure/SASKeyGeneratorInterface.java | 66 + .../hadoop/fs/azure/SecureModeException.java | 40 + .../fs/azure/SecureStorageInterfaceImpl.java | 565 ++++++++ .../hadoop/fs/azure/SelfRenewingLease.java | 2 +- .../hadoop/fs/azure/SendRequestIntercept.java | 3 +- .../hadoop/fs/azure/StorageInterface.java | 28 +- .../fs/azure/WasbRemoteCallException.java | 41 + .../hadoop/fs/azure/WasbRemoteCallHelper.java | 93 ++ .../metrics/AzureFileSystemInstrumentation.java | 2 + .../hadoop-azure/src/site/markdown/index.md | 48 + .../hadoop/fs/azure/TestContainerChecks.java | 12 +- .../fs/azure/TestWasbUriAndConfiguration.java | 17 +- .../src/test/resources/azure-test.xml | 15 +- .../src/site/markdown/DistCp.md.vm | 1 + hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh | 4 +- .../src/main/data/2jobs2min-rumen-jh.json | 6 - .../sls/scheduler/ResourceSchedulerWrapper.java | 11 +- .../sls/scheduler/SLSCapacityScheduler.java | 53 +- .../hadoop-yarn/bin/stop-yarn.sh | 16 +- .../jdiff/Apache_Hadoop_YARN_API_2.6.0.xml | 2 +- .../jdiff/Apache_Hadoop_YARN_API_2.7.2.xml | 2 +- .../api/records/AbstractResourceRequest.java | 55 - .../yarn/api/records/ApplicationTimeout.java | 12 +- .../hadoop/yarn/api/records/ContainerState.java | 2 + .../yarn/api/records/ContainerUpdateType.java | 9 +- .../hadoop/yarn/api/records/QueueState.java | 11 +- .../yarn/api/records/ResourceRequest.java | 19 +- .../yarn/api/records/UpdateContainerError.java | 6 + .../api/records/UpdateContainerRequest.java | 39 +- .../records/timelineservice/FlowRunEntity.java | 4 +- .../org/apache/hadoop/yarn/conf/HAUtil.java | 4 +- .../hadoop/yarn/conf/YarnConfiguration.java | 136 +- .../src/main/proto/yarn_protos.proto | 1 + .../src/main/proto/yarn_service_protos.proto | 3 +- .../distributedshell/ApplicationMaster.java | 2 +- .../UnmanagedAMLauncher.java | 10 +- .../api/async/impl/AMRMClientAsyncImpl.java | 30 +- .../hadoop/yarn/client/cli/ApplicationCLI.java | 16 +- .../apache/hadoop/yarn/client/cli/LogsCLI.java | 2 +- .../hadoop/yarn/client/cli/RMAdminCLI.java | 2 +- .../apache/hadoop/yarn/client/cli/TopCLI.java | 15 +- ...stHedgingRequestRMFailoverProxyProvider.java | 57 +- ...nagerAdministrationProtocolPBClientImpl.java | 33 +- .../api/async/impl/TestAMRMClientAsync.java | 2 +- .../yarn/client/api/impl/TestAMRMClient.java | 2 +- .../api/impl/TestAMRMClientOnRMRestart.java | 11 +- .../yarn/client/api/impl/TestAMRMProxy.java | 37 +- .../api/impl/TestDistributedScheduling.java | 1 - .../hadoop/yarn/client/cli/TestLogsCLI.java | 15 + .../hadoop/yarn/client/cli/TestYarnCLI.java | 77 +- .../impl/pb/AllocateResponsePBImpl.java | 4 +- .../client/api/impl/TimelineClientImpl.java | 2 +- .../hadoop/yarn/event/AsyncDispatcher.java | 20 +- .../apache/hadoop/yarn/event/Dispatcher.java | 2 +- .../nodelabels/CommonNodeLabelsManager.java | 2 +- .../apache/hadoop/yarn/security/AccessType.java | 2 + .../org/apache/hadoop/yarn/state/Graph.java | 2 +- .../yarn/state/VisualizeStateMachine.java | 4 +- .../yarn/util/ProcfsBasedProcessTree.java | 5 +- .../src/main/resources/yarn-default.xml | 95 +- .../hadoop/yarn/api/BasePBImplRecordsTest.java | 2 +- .../hadoop/yarn/event/DrainDispatcher.java | 6 +- .../hadoop/yarn/event/InlineDispatcher.java | 2 +- .../nodelabels/TestCommonNodeLabelsManager.java | 2 +- .../server/timeline/LeveldbTimelineStore.java | 33 +- .../timeline/TestLeveldbTimelineStore.java | 39 + .../server/api/protocolrecords/RemoteNode.java | 7 + .../OpportunisticContainerAllocator.java | 77 +- .../OpportunisticContainerContext.java | 40 +- .../server/scheduler/SchedulerRequestKey.java | 162 +++ .../hadoop/yarn/server/webapp/AppBlock.java | 9 + .../src/CMakeLists.txt | 16 + .../server/nodemanager/ContainerExecutor.java | 11 +- .../yarn/server/nodemanager/NodeManager.java | 4 +- .../nodemanager/amrmproxy/AMRMProxyService.java | 2 +- .../containermanager/ContainerManagerImpl.java | 6 +- .../application/ApplicationImpl.java | 2 +- .../container/ContainerImpl.java | 2 +- .../linux/resources/TrafficController.java | 13 +- .../localizer/LocalizedResource.java | 2 +- .../localizer/ResourceLocalizationService.java | 1 + .../logaggregation/AppLogAggregatorImpl.java | 4 +- .../scheduler/ContainerScheduler.java | 12 +- .../nodemanager/metrics/NodeManagerMetrics.java | 65 +- .../scheduler/DistributedScheduler.java | 11 +- .../timelineservice/NMTimelinePublisher.java | 2 +- .../nodemanager/webapp/NMWebAppFilter.java | 8 + .../server/nodemanager/webapp/WebServer.java | 35 +- .../impl/container-executor.c | 12 +- .../nodemanager/TestNodeHealthService.java | 2 +- .../nodemanager/TestNodeStatusUpdater.java | 4 +- .../launcher/TestContainerLaunch.java | 73 +- .../TestResourceLocalizationService.java | 2 +- .../TestAppLogAggregatorImpl.java | 4 +- .../TestLogAggregationService.java | 2 +- .../nodemanager/webapp/TestNMWebFilter.java | 176 +++ .../conf/capacity-scheduler.xml | 9 + ...ActiveStandbyElectorBasedElectorService.java | 271 ++++ .../server/resourcemanager/AdminService.java | 83 +- .../ApplicationMasterService.java | 444 +++--- .../server/resourcemanager/ClientRMService.java | 13 +- .../CuratorBasedElectorService.java | 139 ++ .../server/resourcemanager/EmbeddedElector.java | 41 + .../resourcemanager/EmbeddedElectorService.java | 260 ---- .../resourcemanager/LeaderElectorService.java | 129 -- .../resourcemanager/NMLivelinessMonitor.java | 3 +- ...pportunisticContainerAllocatorAMService.java | 137 +- .../server/resourcemanager/RMAppManager.java | 40 +- .../yarn/server/resourcemanager/RMContext.java | 6 +- .../server/resourcemanager/RMContextImpl.java | 15 +- .../server/resourcemanager/RMServerUtils.java | 144 +- .../server/resourcemanager/ResourceManager.java | 41 +- .../ahs/RMApplicationHistoryWriter.java | 4 +- .../metrics/AbstractSystemMetricsPublisher.java | 4 +- .../recovery/FileSystemRMStateStore.java | 2 +- .../recovery/MemoryRMStateStore.java | 2 +- .../resourcemanager/recovery/RMStateStore.java | 2 +- .../reservation/InMemoryPlan.java | 4 +- .../reservation/ReservationInputValidator.java | 2 +- .../server/resourcemanager/rmapp/RMAppImpl.java | 31 +- .../rmapp/attempt/AMLivelinessMonitor.java | 3 +- .../rmapp/attempt/RMAppAttemptImpl.java | 33 +- .../rmapp/monitor/RMAppLifetimeMonitor.java | 4 +- .../rmcontainer/RMContainer.java | 3 +- .../rmcontainer/RMContainerImpl.java | 34 +- .../rmcontainer/RMContainerReservedEvent.java | 2 +- .../resourcemanager/rmnode/RMNodeImpl.java | 2 +- .../scheduler/AbstractYarnScheduler.java | 138 +- .../resourcemanager/scheduler/Allocation.java | 26 +- .../scheduler/AppSchedulingInfo.java | 563 +++----- .../scheduler/ContainerUpdateContext.java | 267 ++++ .../scheduler/ContainerUpdates.java | 68 + .../scheduler/QueueStateManager.java | 100 ++ .../scheduler/SchedulerApplicationAttempt.java | 216 ++- .../scheduler/SchedulerNode.java | 89 +- .../scheduler/SchedulerQueue.java | 69 + .../scheduler/SchedulerQueueManager.java | 3 +- .../scheduler/SchedulerRequestKey.java | 122 -- .../scheduler/SchedulerUtils.java | 42 +- .../scheduler/YarnScheduler.java | 24 +- .../scheduler/capacity/AbstractCSQueue.java | 164 ++- .../AppPriorityACLConfigurationParser.java | 219 +++ .../scheduler/capacity/AppPriorityACLGroup.java | 108 ++ .../scheduler/capacity/CSQueue.java | 4 +- .../scheduler/capacity/CapacityScheduler.java | 178 ++- .../CapacitySchedulerConfiguration.java | 44 +- .../capacity/CapacitySchedulerContext.java | 9 + .../capacity/CapacitySchedulerQueueManager.java | 36 +- .../scheduler/capacity/LeafQueue.java | 44 +- .../scheduler/capacity/ParentQueue.java | 27 +- .../allocator/AbstractContainerAllocator.java | 4 + .../allocator/IncreaseContainerAllocator.java | 2 +- .../allocator/RegularContainerAllocator.java | 138 +- .../scheduler/common/PendingAsk.java | 57 + .../scheduler/common/SchedulerContainer.java | 2 +- .../scheduler/common/fica/FiCaSchedulerApp.java | 71 +- .../common/fica/FiCaSchedulerNode.java | 6 +- .../distributed/NodeQueueLoadMonitor.java | 16 +- .../fair/AllocationFileLoaderService.java | 2 +- .../scheduler/fair/FSAppAttempt.java | 166 +-- .../scheduler/fair/FSPreemptionThread.java | 11 +- .../scheduler/fair/FSQueueMetrics.java | 2 +- .../scheduler/fair/FSSchedulerNode.java | 2 +- .../scheduler/fair/FairScheduler.java | 50 +- .../fair/policies/FairSharePolicy.java | 19 +- .../scheduler/fifo/FifoAppAttempt.java | 18 +- .../scheduler/fifo/FifoScheduler.java | 141 +- .../LocalitySchedulingPlacementSet.java | 414 ++++++ .../placement/SchedulingPlacementSet.java | 94 +- .../security/AppPriorityACLsManager.java | 230 +++ .../webapp/CapacitySchedulerPage.java | 2 +- .../resourcemanager/webapp/NodesPage.java | 6 +- .../server/resourcemanager/webapp/RMWebApp.java | 3 +- .../resourcemanager/webapp/dao/AppInfo.java | 8 +- .../resourcemanager/webapp/dao/ClusterInfo.java | 2 +- .../resourcemanager/webapp/dao/NodeInfo.java | 10 +- .../server/resourcemanager/ACLsTestBase.java | 5 +- .../server/resourcemanager/Application.java | 6 +- .../yarn/server/resourcemanager/MockAM.java | 7 + .../yarn/server/resourcemanager/MockNM.java | 6 + .../yarn/server/resourcemanager/MockRM.java | 35 +- .../server/resourcemanager/RMHATestBase.java | 4 +- .../yarn/server/resourcemanager/Task.java | 2 +- .../TestApplicationMasterService.java | 5 +- .../resourcemanager/TestClientRMService.java | 13 +- .../TestContainerResourceUsage.java | 8 +- .../TestLeaderElectorService.java | 17 +- ...pportunisticContainerAllocatorAMService.java | 456 +++++- .../yarn/server/resourcemanager/TestRM.java | 7 +- .../resourcemanager/TestRMAdminService.java | 56 +- .../resourcemanager/TestRMEmbeddedElector.java | 49 +- .../yarn/server/resourcemanager/TestRMHA.java | 45 +- .../resourcemanager/TestRMNodeTransitions.java | 2 +- .../server/resourcemanager/TestRMRestart.java | 14 +- ...alCapacityPreemptionPolicyMockFramework.java | 8 +- ...estProportionalCapacityPreemptionPolicy.java | 6 +- .../recovery/RMStateStoreTestBase.java | 2 +- .../TestReservationInputValidator.java | 16 +- .../rmapp/TestNodesListManager.java | 3 +- .../attempt/TestRMAppAttemptTransitions.java | 19 +- .../rmcontainer/TestRMContainerImpl.java | 7 +- .../scheduler/TestAbstractYarnScheduler.java | 129 +- .../scheduler/TestAppSchedulingInfo.java | 5 +- .../TestSchedulerApplicationAttempt.java | 5 +- .../scheduler/TestSchedulerUtils.java | 8 +- .../TestApplicationLimitsByPartition.java | 6 + .../capacity/TestApplicationPriority.java | 25 +- ...TestApplicationPriorityACLConfiguration.java | 120 ++ .../capacity/TestApplicationPriorityACLs.java | 206 +++ .../capacity/TestCapacityScheduler.java | 279 ++-- .../scheduler/capacity/TestChildQueueOrder.java | 12 +- .../capacity/TestContainerAllocation.java | 50 + .../scheduler/capacity/TestLeafQueue.java | 80 +- .../TestNodeLabelContainerAllocation.java | 65 +- .../scheduler/capacity/TestParentQueue.java | 6 +- .../scheduler/capacity/TestQueueState.java | 104 ++ .../capacity/TestQueueStateManager.java | 162 +++ .../scheduler/capacity/TestReservations.java | 33 +- .../scheduler/capacity/TestUtils.java | 5 +- .../scheduler/fair/FairSchedulerTestBase.java | 14 +- .../fair/TestContinuousScheduling.java | 11 +- .../scheduler/fair/TestFSAppAttempt.java | 2 +- .../scheduler/fair/TestFairScheduler.java | 35 +- .../scheduler/fifo/TestFifoScheduler.java | 48 +- .../resourcemanager/webapp/TestRMWebApp.java | 2 +- .../webapp/TestRMWebServiceAppsNodelabel.java | 2 +- .../webapp/TestRMWebServicesNodes.java | 10 +- .../hadoop/yarn/server/MiniYARNCluster.java | 10 +- .../hadoop/yarn/server/TestMiniYarnCluster.java | 80 +- .../pom.xml | 142 +- ...stTimelineReaderWebServicesHBaseStorage.java | 6 +- ...TestPhoenixOfflineAggregationWriterImpl.java | 161 --- .../flow/TestHBaseStorageFlowActivity.java | 12 +- .../storage/flow/TestHBaseStorageFlowRun.java | 14 +- .../flow/TestHBaseStorageFlowRunCompaction.java | 44 +- .../hadoop-yarn-server-timelineservice/pom.xml | 30 +- .../collector/TimelineCollectorManager.java | 26 +- .../reader/TimelineReaderServer.java | 26 +- .../PhoenixOfflineAggregationWriterImpl.java | 358 ----- .../storage/TimelineSchemaCreator.java | 22 - .../storage/common/AppIdKeyConverter.java | 5 +- .../common/HBaseTimelineStorageUtils.java | 243 ++++ .../storage/common/TimelineStorageUtils.java | 207 --- .../storage/flow/FlowActivityColumnPrefix.java | 10 +- .../storage/flow/FlowActivityRowKey.java | 4 +- .../storage/flow/FlowRunColumn.java | 6 +- .../storage/flow/FlowRunColumnPrefix.java | 6 +- .../storage/flow/FlowRunCoprocessor.java | 6 +- .../storage/flow/FlowScanner.java | 13 +- .../collector/TestTimelineCollectorManager.java | 74 + .../reader/TestTimelineReaderServer.java | 43 + .../storage/common/TestRowKeys.java | 2 +- .../site/markdown/OpportunisticContainers.md | 225 +++ .../src/site/markdown/ResourceManagerRest.md | 252 +++- .../src/site/markdown/TimelineServiceV2.md | 2 +- .../src/site/markdown/YarnCommands.md | 7 + .../webapp/app/components/app-timeout-bar.js | 60 + .../src/main/webapp/app/models/yarn-app.js | 2 + .../src/main/webapp/app/serializers/yarn-app.js | 21 +- .../templates/components/app-timeout-bar.hbs | 34 + .../src/main/webapp/app/templates/yarn-app.hbs | 10 +- .../src/main/webapp/app/utils/converter.js | 3 + .../components/app-timeout-bar-test.js | 34 + pom.xml | 4 +- 617 files changed, 26395 insertions(+), 8108 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c61010d/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c61010d/hadoop-project/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c61010d/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c61010d/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c61010d/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c61010d/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java ---------------------------------------------------------------------- diff --cc hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java index 15604c6,567bacb..fa55586 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java @@@ -21,13 -21,12 +21,15 @@@ package org.apache.hadoop.fs.s3a import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileContext; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.fs.s3a.scale.S3AScaleTestBase; import org.junit.Assert; + import org.junit.Assume; import org.junit.internal.AssumptionViolatedException; import org.slf4j.Logger; + import org.slf4j.LoggerFactory; import java.io.IOException; import java.net.URI; @@@ -532,61 -502,46 +522,104 @@@ public final class S3ATestUtils private S3ATestUtils() { } + public static void verifyFileStatus(FileStatus status, long size, + long blockSize, long modTime) { + verifyFileStatus(status, size, 0, modTime, 0, blockSize, null, null, null); + } + + public static void verifyFileStatus(FileStatus status, long size, + int replication, + long modTime, + long accessTime, + long blockSize, + String owner, + String group, + FsPermission permission) { + String details = status.toString(); + assertFalse("Not a dir: " + details, status.isDirectory()); + assertEquals("Mod time: " + details, modTime, status.getModificationTime()); + assertEquals("File size: " + details, size, status.getLen()); + assertEquals("Block size: " + details, blockSize, status.getBlockSize()); + if (replication > 0) { + assertEquals("Replication value: " + details, replication, + status.getReplication()); + } + if (accessTime != 0) { + assertEquals("Access time: " + details, accessTime, + status.getAccessTime()); + } + if (owner != null) { + assertEquals("Owner: " + details, owner, status.getOwner()); + } + if (group != null) { + assertEquals("Group: " + details, group, status.getGroup()); + } + if (permission != null) { + assertEquals("Permission: " + details, permission, + status.getPermission()); + } + } + + public static void verifyDirStatus(FileStatus status, + int replication, + long modTime, + long accessTime, + String owner, + String group, + FsPermission permission) { + String details = status.toString(); + assertTrue("Is a dir: " + details, status.isDirectory()); + assertEquals("zero length: " + details, 0, status.getLen()); + + assertEquals("Mod time: " + details, modTime, status.getModificationTime()); + assertEquals("Replication value: " + details, replication, + status.getReplication()); + assertEquals("Access time: " + details, accessTime, status.getAccessTime()); + assertEquals("Owner: " + details, owner, status.getOwner()); + assertEquals("Group: " + details, group, status.getGroup()); + assertEquals("Permission: " + details, permission, status.getPermission()); + } ++ + /** + * Set a bucket specific property to a particular value. + * If the generic key passed in has an {@code fs.s3a. prefix}, + * that's stripped off, so that when the the bucket properties are propagated + * down to the generic values, that value gets copied down. + * @param conf configuration to set + * @param bucket bucket name + * @param genericKey key; can start with "fs.s3a." + * @param value value to set + */ + public static void setBucketOption(Configuration conf, String bucket, + String genericKey, String value) { + final String baseKey = genericKey.startsWith(FS_S3A_PREFIX) ? + genericKey.substring(FS_S3A_PREFIX.length()) + : genericKey; + conf.set(FS_S3A_BUCKET_PREFIX + bucket + '.' + baseKey, value); + } + + /** + * Assert that a configuration option matches the expected value. + * @param conf configuration + * @param key option key + * @param expected expected value + */ + public static void assertOptionEquals(Configuration conf, + String key, + String expected) { + assertEquals("Value of " + key, expected, conf.get(key)); + } + + /** + * Assume that a condition is met. If not: log at WARN and + * then throw an {@link AssumptionViolatedException}. + * @param message + * @param condition + */ + public static void assume(String message, boolean condition) { + if (!condition) { + LOG.warn(message); + } + Assume.assumeTrue(message, condition); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
