Merge branch 'trunk' into HADOOP-13345 branch
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0abbb702 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0abbb702 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0abbb702 Branch: refs/heads/HADOOP-13345 Commit: 0abbb7029a2f7dcd91fda067921335a535b4c9dd Parents: 95e0143 5f5b031 Author: Steve Loughran <ste...@apache.org> Authored: Mon Feb 27 15:13:34 2017 +0000 Committer: Steve Loughran <ste...@apache.org> Committed: Mon Feb 27 15:13:34 2017 +0000 ---------------------------------------------------------------------- .../apache/hadoop/fs/ChecksumFileSystem.java | 2 + .../fs/CommonConfigurationKeysPublic.java | 15 + .../java/org/apache/hadoop/fs/FileSystem.java | 11 + .../org/apache/hadoop/fs/ftp/FTPFileSystem.java | 1 + .../apache/hadoop/io/compress/GzipCodec.java | 78 +-- .../apache/hadoop/io/erasurecode/CodecUtil.java | 28 +- .../io/erasurecode/ErasureCodeConstants.java | 8 +- .../erasurecode/coder/HHXORErasureDecoder.java | 2 +- .../erasurecode/coder/HHXORErasureEncoder.java | 2 +- .../io/erasurecode/coder/RSErasureDecoder.java | 2 +- .../io/erasurecode/coder/RSErasureEncoder.java | 2 +- .../apache/hadoop/io/retry/RetryPolicies.java | 3 +- .../security/ShellBasedUnixGroupsMapping.java | 114 +++- .../main/java/org/apache/hadoop/util/Shell.java | 19 +- .../src/main/resources/core-default.xml | 17 +- .../org/apache/hadoop/cli/CLITestHelper.java | 15 +- .../erasurecode/TestCodecRawCoderMapping.java | 10 +- .../coder/TestHHXORErasureCoder.java | 2 +- .../erasurecode/coder/TestRSErasureCoder.java | 2 +- .../hadoop/security/TestGroupsCaching.java | 19 +- .../TestShellBasedUnixGroupsMapping.java | 135 ++++- .../hdfs/client/HdfsClientConfigKeys.java | 2 + .../hadoop/hdfs/web/WebHdfsFileSystem.java | 6 +- .../hdfs/web/resources/AclPermissionParam.java | 17 +- .../hadoop-hdfs/src/main/bin/hdfs | 2 +- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 16 + .../qjournal/client/QuorumJournalManager.java | 38 +- .../hadoop/hdfs/qjournal/server/JNStorage.java | 9 +- .../hadoop/hdfs/qjournal/server/Journal.java | 19 + .../hdfs/qjournal/server/JournalNode.java | 23 +- .../hdfs/qjournal/server/JournalNodeSyncer.java | 413 +++++++++++++++ .../BlockPlacementPolicyDefault.java | 2 +- .../blockmanagement/DatanodeStorageInfo.java | 5 + .../hadoop/hdfs/server/common/Storage.java | 9 + .../apache/hadoop/hdfs/server/common/Util.java | 46 +- .../datanode/web/webhdfs/WebHdfsHandler.java | 10 +- .../hadoop/hdfs/server/namenode/NNStorage.java | 5 +- .../server/namenode/NameNodeHttpServer.java | 4 + .../hdfs/server/namenode/TransferFsImage.java | 3 +- .../org/apache/hadoop/hdfs/tools/ECAdmin.java | 320 ++++++++++++ .../hadoop/hdfs/tools/StoragePolicyAdmin.java | 4 +- .../hadoop/hdfs/tools/erasurecode/ECCli.java | 62 --- .../hdfs/tools/erasurecode/ECCommand.java | 248 --------- .../OfflineImageReconstructor.java | 85 ++- .../offlineImageViewer/PBImageXmlWriter.java | 30 +- .../src/main/resources/hdfs-default.xml | 49 ++ .../src/site/markdown/HDFSErasureCoding.md | 22 +- .../src/site/markdown/HdfsEditsViewer.md | 73 ++- .../hadoop/cli/CLITestCmdErasureCoding.java | 4 +- .../cli/util/ErasureCodingCliCmdExecutor.java | 6 +- .../TestDFSRSDefault10x4StripedInputStream.java | 2 +- ...TestDFSRSDefault10x4StripedOutputStream.java | 2 +- ...fault10x4StripedOutputStreamWithFailure.java | 4 +- .../hadoop/hdfs/TestDFSStripedInputStream.java | 2 +- .../hadoop/hdfs/TestDFSStripedOutputStream.java | 2 +- .../TestDFSStripedOutputStreamWithFailure.java | 2 +- .../hadoop/hdfs/TestReconstructStripedFile.java | 2 +- .../TestUnsetAndChangeDirectoryEcPolicy.java | 2 +- .../hdfs/qjournal/MiniJournalCluster.java | 8 + .../hadoop/hdfs/qjournal/MiniQJMHACluster.java | 1 + .../hdfs/qjournal/TestJournalNodeSync.java | 264 ++++++++++ .../blockmanagement/TestReplicationPolicy.java | 35 +- .../TestReplicationPolicyWithNodeGroup.java | 23 +- .../namenode/TestNameNodeMetricsLogger.java | 11 +- .../TestOfflineImageViewer.java | 15 +- .../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 29 +- .../hadoop/hdfs/web/resources/TestParam.java | 34 ++ .../test/resources/testErasureCodingConf.xml | 137 +++-- .../src/site/markdown/MapReduceTutorial.md | 2 +- .../org/apache/hadoop/mapred/YARNRunner.java | 141 +++-- hadoop-project/pom.xml | 2 +- .../hadoop/fs/s3a/S3ABlockOutputStream.java | 68 ++- .../org/apache/hadoop/fs/s3a/S3ADataBlocks.java | 514 +++++++++++-------- .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 80 ++- .../hadoop/fs/s3a/S3AInstrumentation.java | 40 +- .../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 26 + .../hadoop/fs/s3native/S3xLoginHelper.java | 15 +- .../hadoop/fs/s3a/ITestS3ABlockOutputArray.java | 76 ++- .../fs/s3a/ITestS3ABlockOutputByteBuffer.java | 5 +- .../hadoop/fs/s3a/ITestS3ABlockOutputDisk.java | 12 + .../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 13 + .../apache/hadoop/fs/s3a/TestDataBlocks.java | 45 +- .../fs/s3a/scale/AbstractSTestS3AHugeFiles.java | 28 +- .../hadoop/fs/s3native/TestS3xLoginHelper.java | 28 + .../src/site/markdown/index.md | 237 +++++---- .../java/org/apache/hadoop/fs/azure/Wasbs.java | 47 ++ .../fs/azure/TestWasbUriAndConfiguration.java | 57 ++ .../fs/swift/snative/SwiftNativeFileSystem.java | 2 + .../org/apache/hadoop/yarn/sls/SLSRunner.java | 20 +- .../hadoop/yarn/sls/appmaster/AMSimulator.java | 89 ++-- .../yarn/sls/appmaster/MRAMSimulator.java | 218 ++++---- .../sls/resourcemanager/MockAMLauncher.java | 115 +++++ .../sls/scheduler/SLSCapacityScheduler.java | 24 + .../hadoop/yarn/conf/YarnConfiguration.java | 3 + .../resource/DefaultResourceCalculator.java | 3 +- .../resource/DominantResourceCalculator.java | 13 +- .../yarn/util/resource/ResourceCalculator.java | 32 +- .../src/main/resources/yarn-default.xml | 5 + .../server/resourcemanager/ClientRMService.java | 6 +- .../server/resourcemanager/RMAppManager.java | 5 +- .../reservation/AbstractReservationSystem.java | 8 +- .../planning/AlignedPlannerWithGreedy.java | 16 +- .../planning/GreedyReservationAgent.java | 19 +- .../reservation/planning/PlanningAlgorithm.java | 4 + .../reservation/planning/ReservationAgent.java | 8 + .../planning/TryManyReservationAgents.java | 5 +- .../scheduler/fair/FSAppAttempt.java | 61 ++- .../scheduler/fair/FSContext.java | 21 +- .../scheduler/fair/FSLeafQueue.java | 24 +- .../scheduler/fair/FSPreemptionThread.java | 12 +- .../resourcemanager/scheduler/fair/FSQueue.java | 2 +- .../scheduler/fair/FairScheduler.java | 11 +- .../scheduler/fair/SchedulingPolicy.java | 19 +- .../DominantResourceFairnessPolicy.java | 24 +- .../fair/policies/FairSharePolicy.java | 3 +- .../planning/TestAlignedPlanner.java | 3 + .../planning/TestGreedyReservationAgent.java | 4 +- .../scheduler/fair/TestFSAppStarvation.java | 24 +- .../scheduler/fair/TestFairScheduler.java | 136 +++-- .../fair/TestFairSchedulerPreemption.java | 44 +- .../TestDominantResourceFairnessPolicy.java | 25 +- .../yarn/server/timeline/EntityCacheItem.java | 9 +- 122 files changed, 3602 insertions(+), 1445 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0abbb702/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0abbb702/hadoop-project/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0abbb702/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0abbb702/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0abbb702/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0abbb702/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0abbb702/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java ---------------------------------------------------------------------- diff --cc hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java index 72fb3be,9528967..ec0952d --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java @@@ -20,10 -20,9 +20,11 @@@ package org.apache.hadoop.fs.s3a import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; + import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileContext; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; import org.junit.Assert; import org.junit.Assume; import org.junit.internal.AssumptionViolatedException; --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org