Merge changes from trunk
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/432b8637 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/432b8637 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/432b8637 Branch: refs/heads/trunk Commit: 432b863719e1a36459812cdcbe4785e6a19f3f3a Parents: 2e4033e b6d3230 Author: Jing Zhao <j...@hortonworks.com> Authored: Tue Sep 16 17:58:39 2014 -0700 Committer: Jing Zhao <j...@hortonworks.com> Committed: Tue Sep 16 17:58:39 2014 -0700 ---------------------------------------------------------------------- hadoop-common-project/hadoop-auth/pom.xml | 13 + .../server/AuthenticationFilter.java | 152 ++++-- .../util/RandomSignerSecretProvider.java | 4 +- .../util/RolloverSignerSecretProvider.java | 7 +- .../util/SignerSecretProvider.java | 9 +- .../util/StringSignerSecretProvider.java | 15 +- .../util/ZKSignerSecretProvider.java | 506 +++++++++++++++++++ .../src/site/apt/Configuration.apt.vm | 148 +++++- .../hadoop-auth/src/site/apt/index.apt.vm | 5 + .../server/TestAuthenticationFilter.java | 117 ++++- .../util/TestJaasConfiguration.java | 55 ++ .../util/TestRandomSignerSecretProvider.java | 2 +- .../util/TestRolloverSignerSecretProvider.java | 2 +- .../authentication/util/TestSigner.java | 23 +- .../util/TestStringSignerSecretProvider.java | 9 +- .../util/TestZKSignerSecretProvider.java | 270 ++++++++++ hadoop-common-project/hadoop-common/CHANGES.txt | 16 + .../hadoop-common/src/main/bin/hadoop-config.sh | 1 + .../src/main/bin/hadoop-functions.sh | 127 +++-- .../hadoop-common/src/main/conf/hadoop-env.sh | 81 --- .../src/main/conf/hadoop-policy.xml | 9 + .../main/conf/hadoop-user-functions.sh.example | 94 ++++ .../main/java/org/apache/hadoop/ha/HAAdmin.java | 6 +- .../io/nativeio/SharedFileDescriptorFactory.c | 32 +- .../src/site/apt/CommandsManual.apt.vm | 53 ++ hadoop-common-project/hadoop-kms/pom.xml | 25 +- .../hadoop/crypto/key/kms/server/MiniKMS.java | 47 +- .../test/resources/mini-kms-acls-default.xml | 135 +++++ .../hadoop/fs/http/server/TestHttpFSServer.java | 8 +- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 20 + hadoop-hdfs-project/hadoop-hdfs/pom.xml | 13 + .../hadoop-hdfs/src/main/bin/hdfs-config.sh | 7 +- .../java/org/apache/hadoop/hdfs/DFSClient.java | 5 +- .../org/apache/hadoop/hdfs/DFSOutputStream.java | 40 +- .../hadoop/hdfs/protocol/ClientProtocol.java | 4 +- .../hadoop/hdfs/protocol/EncryptionZone.java | 18 +- .../hdfs/protocol/EncryptionZoneIterator.java | 20 +- .../hdfs/protocol/EncryptionZoneWithId.java | 81 --- .../protocol/EncryptionZoneWithIdIterator.java | 53 -- .../datatransfer/DataTransferProtoUtil.java | 44 +- .../hdfs/protocol/datatransfer/Receiver.java | 123 +++-- .../hdfs/protocol/datatransfer/Sender.java | 26 +- ...tNamenodeProtocolServerSideTranslatorPB.java | 6 +- .../ClientNamenodeProtocolTranslatorPB.java | 14 +- .../apache/hadoop/hdfs/protocolPB/PBHelper.java | 12 +- .../web/resources/DatanodeWebHdfsMethods.java | 13 +- .../server/namenode/EncryptionZoneManager.java | 24 +- .../hdfs/server/namenode/FSDirectory.java | 6 +- .../hdfs/server/namenode/FSNamesystem.java | 10 +- .../hdfs/server/namenode/NameNodeRpcServer.java | 6 +- .../src/main/proto/datatransfer.proto | 8 + .../hadoop-hdfs/src/main/proto/encryption.proto | 6 +- .../hadoop/hdfs/TestDFSClientFailover.java | 5 +- .../apache/hadoop/hdfs/TestEncryptionZones.java | 62 ++- .../hadoop/hdfs/TestEncryptionZonesWithKMS.java | 56 ++ .../blockmanagement/TestBlockManager.java | 56 +- .../server/datanode/SimulatedFSDataset.java | 7 +- .../hdfs/tools/TestDFSHAAdminMiniCluster.java | 20 - .../org/apache/hadoop/tracing/TestTracing.java | 52 +- .../TestTracingShortCircuitLocalRead.java | 97 ++++ hadoop-mapreduce-project/bin/mapred-config.sh | 7 +- hadoop-project/pom.xml | 25 + .../org/apache/hadoop/fs/s3a/Constants.java | 36 +- .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 46 +- .../apache/hadoop/fs/s3a/S3AOutputStream.java | 6 +- hadoop-yarn-project/CHANGES.txt | 15 + .../hadoop-yarn/bin/yarn-config.sh | 11 +- .../hadoop-yarn/conf/yarn-env.sh | 52 +- .../hadoop/yarn/conf/YarnConfiguration.java | 14 + .../applications/distributedshell/Client.java | 18 +- .../TestDSSleepingAppMaster.java | 58 +++ .../distributedshell/TestDistributedShell.java | 76 +++ .../src/main/resources/yarn-default.xml | 10 + .../ApplicationHistoryClientService.java | 15 +- .../authorize/TimelinePolicyProvider.java | 44 ++ .../hadoop-yarn-server-nodemanager/pom.xml | 14 +- .../util/CgroupsLCEResourcesHandler.java | 29 +- .../util/TestCgroupsLCEResourcesHandler.java | 165 +++++- 78 files changed, 2892 insertions(+), 634 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/432b8637/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/432b8637/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/432b8637/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/432b8637/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/432b8637/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/432b8637/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/432b8637/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/432b8637/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/432b8637/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/432b8637/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/432b8637/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java index 0e13e83,1a8262f..9b980dc --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java @@@ -38,16 -40,19 +40,21 @@@ import org.apache.hadoop.fs.Path import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.StorageType; + import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.BlockTargetPair; + import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; + import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; + import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.net.NetworkTopology; +import org.junit.Assert; + import org.apache.hadoop.test.GenericTestUtils; import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; @@@ -601,19 -606,53 +608,66 @@@ public class TestBlockManager new BlockListAsLongs(null, null)); assertEquals(1, ds.getBlockReportCount()); } + - + /** + * Tests that a namenode doesn't choose a datanode with full disks to + * store blocks. + * @throws Exception + */ + @Test + public void testStorageWithRemainingCapacity() throws Exception { + final Configuration conf = new HdfsConfiguration(); + final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); + FileSystem fs = FileSystem.get(conf); + Path file1 = null; + try { + cluster.waitActive(); + final FSNamesystem namesystem = cluster.getNamesystem(); + final String poolId = namesystem.getBlockPoolId(); + final DatanodeRegistration nodeReg = + DataNodeTestUtils.getDNRegistrationForBP(cluster.getDataNodes(). + get(0), poolId); + final DatanodeDescriptor dd = NameNodeAdapter.getDatanode(namesystem, + nodeReg); + // By default, MiniDFSCluster will create 1 datanode with 2 storages. + // Assigning 64k for remaining storage capacity and will + //create a file with 100k. + for(DatanodeStorageInfo storage: dd.getStorageInfos()) { + storage.setUtilizationForTesting(65536, 0, 65536, 0); + } + //sum of the remaining capacity of both the storages + dd.setRemaining(131072); + file1 = new Path("testRemainingStorage.dat"); + try { + DFSTestUtil.createFile(fs, file1, 102400, 102400, 102400, (short)1, + 0x1BAD5EED); + } + catch (RemoteException re) { + GenericTestUtils.assertExceptionContains("nodes instead of " + + "minReplication", re); + } + } + finally { + // Clean up + assertTrue(fs.exists(file1)); + fs.delete(file1, true); + assertTrue(!fs.exists(file1)); + cluster.shutdown(); + } + } -} + @Test + public void testUseDelHint() { + DatanodeStorageInfo delHint = new DatanodeStorageInfo( + DFSTestUtil.getLocalDatanodeDescriptor(), new DatanodeStorage("id")); + List<DatanodeStorageInfo> moreThan1Racks = Arrays.asList(delHint); + List<StorageType> excessTypes = new ArrayList<StorageType>(); + + excessTypes.add(StorageType.DEFAULT); + Assert.assertTrue(BlockManager.useDelHint(true, delHint, null, + moreThan1Racks, excessTypes)); + excessTypes.add(StorageType.SSD); + Assert.assertFalse(BlockManager.useDelHint(true, delHint, null, + moreThan1Racks, excessTypes)); + } - } ++}
