Merge branch 'trunk' into HDFS-7240
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/30473ecf Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/30473ecf Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/30473ecf Branch: refs/heads/HDFS-7240 Commit: 30473ecfb426990fbefc7a37b725c799a25c609a Parents: 643c5e5 8831b18 Author: Anu Engineer <aengin...@apache.org> Authored: Mon Mar 28 10:34:52 2016 -0700 Committer: Anu Engineer <aengin...@apache.org> Committed: Mon Mar 28 10:34:52 2016 -0700 ---------------------------------------------------------------------- BUILDING.txt | 2 +- dev-support/bin/dist-tools-hooks-maker | 182 ++++++++ dev-support/docker/Dockerfile | 8 + .../JWTRedirectAuthenticationHandler.java | 16 +- .../TestJWTRedirectAuthentictionHandler.java | 37 +- .../hadoop-common/src/main/bin/hadoop | 10 +- .../src/main/bin/hadoop-functions.sh | 92 +++- .../src/main/bin/hadoop-layout.sh.example | 10 +- .../hadoop-common/src/main/conf/hadoop-env.sh | 23 +- .../crypto/key/kms/KMSClientProvider.java | 8 +- .../hadoop/crypto/key/kms/ValueQueue.java | 16 +- .../java/org/apache/hadoop/fs/FileSystem.java | 126 ++++- .../java/org/apache/hadoop/net/NetUtils.java | 16 +- .../org/apache/hadoop/net/NetworkTopology.java | 17 +- .../java/org/apache/hadoop/net/NodeBase.java | 18 +- .../src/org/apache/hadoop/io/compress/lz4/lz4.c | 2 +- .../src/main/resources/core-default.xml | 2 +- .../src/site/markdown/GroupsMapping.md | 26 +- .../src/site/markdown/HttpAuthentication.md | 6 +- .../hadoop/crypto/key/TestValueQueue.java | 6 - .../java/org/apache/hadoop/ipc/TestRpcBase.java | 192 +++++++- .../java/org/apache/hadoop/ipc/TestSaslRPC.java | 395 ++++------------ .../hadoop/security/TestDoAsEffectiveUser.java | 6 +- .../hadoop-common/src/test/proto/test.proto | 9 + .../src/test/proto/test_rpc_service.proto | 4 + ...apache.hadoop.security.token.TokenIdentifier | 2 +- .../scripts/hadoop_add_to_classpath_tools.bats | 47 ++ .../hadoop_add_to_classpath_toolspath.bats | 74 --- .../src/test/scripts/hadoop_basic_init.bats | 3 +- .../src/test/scripts/hadoop_bootstrap.bats | 15 +- .../src/test/scripts/hadoop_entry_tests.bats | 49 ++ .../src/test/scripts/hadoop_shellprofile.bats | 7 + .../hadoop/crypto/key/kms/server/KMS.java | 14 +- .../hadoop/crypto/key/kms/server/TestKMS.java | 42 ++ hadoop-dist/pom.xml | 18 + .../dev-support/findbugsExcludeFile.xml | 1 + .../org/apache/hadoop/hdfs/BlockReader.java | 10 +- .../apache/hadoop/hdfs/BlockReaderFactory.java | 7 +- .../apache/hadoop/hdfs/BlockReaderLocal.java | 10 +- .../hadoop/hdfs/BlockReaderLocalLegacy.java | 10 +- .../org/apache/hadoop/hdfs/ClientContext.java | 56 ++- .../java/org/apache/hadoop/hdfs/DFSClient.java | 22 +- .../org/apache/hadoop/hdfs/DFSInputStream.java | 30 +- .../hadoop/hdfs/DFSStripedInputStream.java | 3 - .../apache/hadoop/hdfs/ExternalBlockReader.java | 10 +- .../apache/hadoop/hdfs/FileChecksumHelper.java | 187 +++++++- .../apache/hadoop/hdfs/RemoteBlockReader.java | 29 +- .../apache/hadoop/hdfs/RemoteBlockReader2.java | 29 +- .../org/apache/hadoop/hdfs/ReplicaAccessor.java | 7 + .../hdfs/protocol/DatanodeAdminProperties.java | 100 ++++ .../apache/hadoop/hdfs/protocol/DatanodeID.java | 6 + .../hadoop/hdfs/protocol/StripedBlockInfo.java | 61 +++ .../datatransfer/DataTransferProtocol.java | 16 +- .../hadoop/hdfs/protocol/datatransfer/Op.java | 1 + .../hdfs/protocol/datatransfer/Sender.java | 19 + .../hadoop/hdfs/protocolPB/PBHelperClient.java | 42 +- .../hdfs/util/CombinedHostsFileReader.java | 76 +++ .../hdfs/util/CombinedHostsFileWriter.java | 69 +++ .../hadoop/hdfs/util/StripedBlockUtil.java | 12 + .../src/main/proto/datatransfer.proto | 9 +- .../src/main/native/fuse-dfs/fuse_connect.c | 5 + .../hadoop-hdfs/src/main/bin/hdfs | 5 +- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 8 +- .../hdfs/protocol/datatransfer/Receiver.java | 28 ++ .../CombinedHostFileManager.java | 250 ++++++++++ .../server/blockmanagement/DatanodeManager.java | 59 ++- .../blockmanagement/HostConfigManager.java | 80 ++++ .../server/blockmanagement/HostFileManager.java | 147 ++---- .../hdfs/server/blockmanagement/HostSet.java | 114 +++++ .../server/datanode/BlockChecksumHelper.java | 284 +++++++++--- .../hdfs/server/datanode/BlockReceiver.java | 2 +- .../hdfs/server/datanode/DataXceiver.java | 43 ++ .../erasurecode/ErasureCodingWorker.java | 3 +- .../server/namenode/EncryptionZoneManager.java | 15 + .../server/namenode/FSDirEncryptionZoneOp.java | 83 ++++ .../hdfs/server/namenode/FSNamesystem.java | 27 ++ .../OfflineImageReconstructor.java | 312 +++++++------ .../offlineImageViewer/PBImageXmlWriter.java | 463 +++++++++++++------ .../src/main/resources/hdfs-default.xml | 32 ++ .../src/main/webapps/hdfs/dfshealth.html | 13 +- .../src/site/markdown/HdfsUserGuide.md | 6 +- .../hadoop-hdfs/src/site/markdown/WebHDFS.md | 61 +-- .../hadoop/fs/TestEnhancedByteBufferAccess.java | 4 +- .../hadoop/hdfs/TestBlockReaderLocal.java | 4 +- .../org/apache/hadoop/hdfs/TestConnCache.java | 2 - .../apache/hadoop/hdfs/TestDatanodeReport.java | 57 ++- .../hadoop/hdfs/TestDistributedFileSystem.java | 62 +++ .../apache/hadoop/hdfs/TestEncryptionZones.java | 4 +- .../hadoop/hdfs/TestEncryptionZonesWithKMS.java | 36 +- .../hadoop/hdfs/TestExternalBlockReader.java | 8 +- .../apache/hadoop/hdfs/TestFileChecksum.java | 247 ++++++++++ .../TestBlocksWithNotEnoughRacks.java | 34 +- .../blockmanagement/TestDatanodeManager.java | 8 +- .../blockmanagement/TestHostFileManager.java | 10 +- .../hdfs/server/namenode/TestHostsFiles.java | 70 ++- .../server/namenode/TestNameNodeMXBean.java | 25 +- .../hdfs/server/namenode/TestStartup.java | 54 +-- .../TestUpgradeDomainBlockPlacementPolicy.java | 169 +++++++ .../hadoop/hdfs/util/HostsFileWriter.java | 122 +++++ .../hdfs/util/TestCombinedHostsFileReader.java | 79 ++++ .../apache/hadoop/net/TestNetworkTopology.java | 7 + .../src/test/resources/dfs.hosts.json | 5 + hadoop-mapreduce-project/bin/mapred | 10 +- .../org/apache/hadoop/mapreduce/TaskID.java | 57 ++- .../org/apache/hadoop/mapreduce/tools/CLI.java | 48 +- .../hadoop/mapreduce/util/ProcessTree.java | 7 +- .../src/site/markdown/EncryptedShuffle.md | 2 +- .../org/apache/hadoop/mapreduce/TestTaskID.java | 461 ++++++++++++++++++ .../apache/hadoop/mapreduce/tools/TestCLI.java | 38 +- .../java/org/apache/hadoop/hdfs/NNBench.java | 1 + .../mapreduce/v2/TestMRJobsWithProfiler.java | 2 +- hadoop-tools/hadoop-archive-logs/pom.xml | 17 + hadoop-tools/hadoop-archives/pom.xml | 17 + hadoop-tools/hadoop-aws/pom.xml | 17 + .../src/site/markdown/tools/hadoop-aws/index.md | 5 +- hadoop-tools/hadoop-azure/pom.xml | 18 + .../hadoop-azure/src/site/markdown/index.md | 3 + hadoop-tools/hadoop-datajoin/pom.xml | 16 + hadoop-tools/hadoop-distcp/pom.xml | 11 + .../java/org/apache/hadoop/tools/DistCp.java | 79 ---- .../apache/hadoop/tools/DistCpConstants.java | 16 - .../apache/hadoop/tools/DistCpOptionSwitch.java | 9 - .../org/apache/hadoop/tools/DistCpOptions.java | 22 - .../org/apache/hadoop/tools/OptionsParser.java | 5 - .../apache/hadoop/tools/mapred/CopyMapper.java | 68 --- .../src/site/markdown/DistCp.md.vm | 39 -- .../apache/hadoop/tools/TestOptionsParser.java | 17 +- hadoop-tools/hadoop-extras/pom.xml | 17 + hadoop-tools/hadoop-gridmix/pom.xml | 17 + hadoop-tools/hadoop-kafka/pom.xml | 17 + hadoop-tools/hadoop-openstack/pom.xml | 17 + .../hadoop-openstack/src/site/markdown/index.md | 7 +- hadoop-tools/hadoop-rumen/pom.xml | 17 + hadoop-tools/hadoop-sls/pom.xml | 17 + .../hadoop-sls/src/main/bin/rumen2sls.sh | 2 +- hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh | 2 +- hadoop-tools/hadoop-streaming/pom.xml | 17 + hadoop-yarn-project/hadoop-yarn/bin/yarn | 4 +- .../hadoop/yarn/conf/YarnConfiguration.java | 6 + .../hadoop/yarn/client/TestRMFailover.java | 7 +- .../yarn/client/api/impl/TestAMRMProxy.java | 413 +++++++++++++++++ .../api/impl/FileSystemTimelineWriter.java | 106 ++++- .../yarn/util/ProcfsBasedProcessTree.java | 6 +- .../util/ResourceCalculatorProcessTree.java | 8 +- .../yarn/util/WindowsBasedProcessTree.java | 5 +- .../yarn/util/TestProcfsBasedProcessTree.java | 8 - .../util/TestResourceCalculatorProcessTree.java | 3 +- .../yarn/util/TestWindowsBasedProcessTree.java | 3 +- .../nodemanager/amrmproxy/AMRMProxyService.java | 14 +- .../amrmproxy/DefaultRequestInterceptor.java | 7 + .../containermanager/ContainerManagerImpl.java | 49 +- .../server/resourcemanager/ClientRMService.java | 9 +- .../resourcemanager/webapp/RMWebAppFilter.java | 40 +- .../webapp/dao/ReservationIdInfo.java | 64 --- .../webapp/dao/ReservationInfo.java | 7 +- .../ParameterizedSchedulerTestBase.java | 36 +- .../yarn/server/resourcemanager/TestRM.java | 4 - .../server/resourcemanager/TestRMRestart.java | 4 - .../TestWorkPreservingRMRestart.java | 9 - .../reservation/TestReservationSystem.java | 4 - .../scheduler/TestAbstractYarnScheduler.java | 6 - .../security/TestClientToAMTokens.java | 4 - .../webapp/TestRMWebServicesReservation.java | 15 +- .../src/test/resources/test-fair-scheduler.xml | 34 +- .../hadoop/yarn/server/MiniYARNCluster.java | 75 ++- .../src/site/markdown/CapacityScheduler.md | 12 + .../src/site/markdown/FairScheduler.md | 6 +- .../src/site/markdown/ResourceManagerRest.md | 217 +++++++++ 168 files changed, 5848 insertions(+), 1821 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/30473ecf/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java index 30e946d,af720c7..6f5f1e3 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java @@@ -22,8 -22,9 +22,10 @@@ import org.apache.hadoop.classification import org.apache.hadoop.classification.InterfaceStability; import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos; + import java.net.InetSocketAddress; + /** * This class represents the primary identifier for a Datanode. * Datanodes are identified by how they can be contacted (hostname @@@ -277,54 -277,7 +279,58 @@@ public class DatanodeID implements Comp return getXferAddr().compareTo(that.getXferAddr()); } + /** + * Returns the container port. + * @return Port + */ + public int getContainerPort() { + return containerPort; + } + + /** + * Sets the container port. + * @param containerPort - container port. + */ + public void setContainerPort(int containerPort) { + this.containerPort = containerPort; + } + + /** + * Returns a DataNode ID from the protocol buffers. + * + * @param datanodeIDProto - protoBuf Message + * @return DataNodeID + */ + public static DatanodeID getFromProtoBuf(HdfsProtos.DatanodeIDProto + datanodeIDProto) { + DatanodeID id = new DatanodeID(datanodeIDProto.getDatanodeUuid(), + datanodeIDProto.getIpAddr(), datanodeIDProto.getHostName(), + datanodeIDProto.getXferPort(), datanodeIDProto.getInfoPort(), + datanodeIDProto.getInfoSecurePort(), datanodeIDProto.getIpcPort()); + id.setContainerPort(datanodeIDProto.getContainerPort()); + return id; + } + + /** + * Returns a DataNodeID protobuf message from a datanode ID. + * @return HdfsProtos.DatanodeIDProto + */ + public HdfsProtos.DatanodeIDProto getProtoBufMessage() { + HdfsProtos.DatanodeIDProto.Builder builder = + HdfsProtos.DatanodeIDProto.newBuilder(); + + return builder.setDatanodeUuid(this.getDatanodeUuid()) + .setIpAddr(this.getIpcAddr()) + .setHostName(this.getHostName()) + .setXferPort(this.getXferPort()) + .setInfoPort(this.getInfoPort()) + .setInfoSecurePort(this.getInfoSecurePort()) + .setIpcPort(this.getIpcPort()) + .setContainerPort(this.getContainerPort()) + .build(); + } ++ + public InetSocketAddress getResolvedAddress() { + return new InetSocketAddress(this.getIpAddr(), this.getXferPort()); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/30473ecf/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs ----------------------------------------------------------------------