Merge remote-tracking branch 'apache/trunk' into YARN-2928
Conflicts:
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fb1b5960
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fb1b5960
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fb1b5960
Branch: refs/heads/YARN-2928
Commit: fb1b596006c7b59b6b72f3974e37e67f414f5667
Parents: 821b68d 8212877
Author: Zhijie Shen <zjs...@apache.org>
Authored: Thu Mar 12 15:19:47 2015 -0700
Committer: Zhijie Shen <zjs...@apache.org>
Committed: Thu Mar 12 15:38:09 2015 -0700
----------------------------------------------------------------------
.gitignore | 1 +
BUILDING.txt | 9 +-
hadoop-common-project/hadoop-common/CHANGES.txt | 53 ++
.../src/main/bin/hadoop-daemons.sh | 13 +-
.../src/main/bin/hadoop-functions.sh | 18 +-
.../src/main/conf/shellprofile.d/example | 106 ----
.../src/main/conf/shellprofile.d/example.sh | 106 ++++
.../fs/CommonConfigurationKeysPublic.java | 6 +-
.../apache/hadoop/fs/DelegateToFileSystem.java | 3 +-
.../io/compress/bzip2/Bzip2Compressor.java | 3 -
.../io/compress/bzip2/Bzip2Decompressor.java | 3 -
.../hadoop/io/compress/lz4/Lz4Compressor.java | 4 -
.../hadoop/io/compress/lz4/Lz4Decompressor.java | 4 -
.../io/compress/snappy/SnappyCompressor.java | 4 -
.../io/compress/snappy/SnappyDecompressor.java | 4 -
.../hadoop/io/compress/zlib/ZlibCompressor.java | 3 -
.../io/compress/zlib/ZlibDecompressor.java | 5 +-
.../main/java/org/apache/hadoop/ipc/Client.java | 33 +-
.../main/java/org/apache/hadoop/ipc/Server.java | 9 +-
.../hadoop/security/LdapGroupsMapping.java | 54 +-
.../hadoop/io/compress/bzip2/Bzip2Compressor.c | 9 +-
.../io/compress/bzip2/Bzip2Decompressor.c | 7 -
.../hadoop/io/compress/lz4/Lz4Compressor.c | 13 -
.../hadoop/io/compress/lz4/Lz4Decompressor.c | 8 -
.../io/compress/snappy/SnappyCompressor.c | 8 -
.../io/compress/snappy/SnappyDecompressor.c | 8 -
.../hadoop/io/compress/zlib/ZlibCompressor.c | 11 -
.../hadoop/io/compress/zlib/ZlibDecompressor.c | 10 -
.../src/main/resources/core-default.xml | 20 +
.../hadoop-common/src/main/winutils/task.c | 144 ++++-
.../src/main/winutils/win8sdk.props | 28 +
.../src/main/winutils/winutils.vcxproj | 3 +
.../hadoop-common/src/site/markdown/Metrics.md | 2 +
.../hadoop/fs/TestDelegateToFileSystem.java | 52 ++
.../apache/hadoop/io/compress/TestCodec.java | 16 +-
.../bzip2/TestBzip2CompressorDecompressor.java | 104 ++++
.../lz4/TestLz4CompressorDecompressor.java | 17 +
.../TestSnappyCompressorDecompressor.java | 17 +
.../zlib/TestZlibCompressorDecompressor.java | 17 +
.../hadoop/security/TestLdapGroupsMapping.java | 42 +-
.../security/TestLdapGroupsMappingBase.java | 77 +++
.../TestLdapGroupsMappingWithPosixGroup.java | 103 +++
.../org/apache/hadoop/util/TestWinUtils.java | 62 ++
.../java/org/apache/hadoop/minikdc/MiniKdc.java | 17 +-
.../minikdc/TestChangeOrgNameAndDomain.java | 32 +
hadoop-dist/pom.xml | 89 ++-
.../hadoop/hdfs/nfs/conf/NfsConfigKeys.java | 14 +
.../hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java | 12 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 41 ++
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 6 +-
.../apache/hadoop/hdfs/HdfsConfiguration.java | 2 +-
.../server/blockmanagement/BlockManager.java | 121 +---
.../server/blockmanagement/DatanodeManager.java | 109 +---
.../blockmanagement/DecommissionManager.java | 619 +++++++++++++++++--
.../hadoop/hdfs/server/common/Storage.java | 2 +-
.../hdfs/server/datanode/BPServiceActor.java | 2 +
.../hadoop/hdfs/server/datanode/DataNode.java | 109 +++-
.../hdfs/server/datanode/DataStorage.java | 15 +-
.../hdfs/server/datanode/DirectoryScanner.java | 20 +-
.../server/datanode/fsdataset/FsDatasetSpi.java | 16 +-
.../impl/FsDatasetAsyncDiskService.java | 31 +-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 153 +++--
.../datanode/fsdataset/impl/FsVolumeImpl.java | 3 +-
.../datanode/fsdataset/impl/FsVolumeList.java | 41 +-
.../datanode/metrics/DataNodeMetrics.java | 5 +
.../datanode/web/webhdfs/ParameterParser.java | 2 +-
.../hadoop/hdfs/web/resources/OffsetParam.java | 5 +
.../src/main/resources/hdfs-default.xml | 23 +-
.../hadoop-hdfs/src/main/shellprofile.d/hdfs | 36 --
.../hadoop-hdfs/src/main/shellprofile.d/hdfs.sh | 36 ++
.../src/site/markdown/HdfsNfsGateway.md | 44 +-
.../src/site/xdoc/HdfsRollingUpgrade.xml | 4 +-
.../hadoop/hdfs/TestAppendSnapshotTruncate.java | 61 +-
.../hadoop/hdfs/TestBlockReaderLocalLegacy.java | 63 ++
.../apache/hadoop/hdfs/TestDecommission.java | 412 ++++++++----
.../blockmanagement/BlockManagerTestUtil.java | 8 +-
.../TestReplicationPolicyConsiderLoad.java | 2 +-
.../server/datanode/SimulatedFSDataset.java | 11 +-
.../datanode/TestDataNodeHotSwapVolumes.java | 99 ++-
.../server/datanode/TestDataNodeMetrics.java | 2 +
.../datanode/TestDataNodeVolumeFailure.java | 70 +++
.../TestDataNodeVolumeFailureReporting.java | 36 +-
.../extdataset/ExternalDatasetImpl.java | 12 +-
.../fsdataset/impl/FsDatasetTestUtil.java | 43 ++
.../fsdataset/impl/TestFsDatasetImpl.java | 118 +++-
.../web/webhdfs/TestParameterParser.java | 19 +
.../namenode/TestDecommissioningStatus.java | 59 +-
.../hdfs/server/namenode/TestFileTruncate.java | 11 +-
.../hadoop/hdfs/server/namenode/TestFsck.java | 2 +-
.../namenode/TestNamenodeCapacityReport.java | 4 +-
hadoop-mapreduce-project/CHANGES.txt | 26 +-
.../apache/hadoop/mapreduce/MRJobConfig.java | 5 +
.../lib/output/FileOutputCommitter.java | 119 ++--
.../src/main/resources/mapred-default.xml | 62 ++
.../hadoop/mapred/TestFileOutputCommitter.java | 134 +++-
.../lib/output/TestFileOutputCommitter.java | 116 +++-
.../org/apache/hadoop/mapred/YARNRunner.java | 5 +-
.../java/org/apache/hadoop/hdfs/NNBench.java | 4 +-
.../apache/hadoop/mapred/TestYARNRunner.java | 26 +
.../shellprofile.d/mapreduce | 41 --
.../shellprofile.d/mapreduce.sh | 41 ++
hadoop-project/pom.xml | 10 +-
.../org/apache/hadoop/fs/s3a/Constants.java | 6 +-
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 20 +-
.../src/site/markdown/tools/hadoop-aws/index.md | 10 +-
hadoop-tools/hadoop-azure/pom.xml | 7 +-
.../fs/azure/AzureNativeFileSystemStore.java | 92 ++-
.../hadoop/fs/azure/NativeAzureFileSystem.java | 10 +-
.../hadoop/fs/azure/PageBlobFormatHelpers.java | 2 +-
.../hadoop/fs/azure/PageBlobInputStream.java | 8 +-
.../hadoop/fs/azure/PageBlobOutputStream.java | 8 +-
.../hadoop/fs/azure/SelfRenewingLease.java | 6 +-
.../fs/azure/SelfThrottlingIntercept.java | 10 +-
.../hadoop/fs/azure/SendRequestIntercept.java | 16 +-
.../hadoop/fs/azure/StorageInterface.java | 30 +-
.../hadoop/fs/azure/StorageInterfaceImpl.java | 50 +-
.../fs/azure/metrics/ErrorMetricUpdater.java | 8 +-
.../metrics/ResponseReceivedMetricUpdater.java | 10 +-
.../fs/azure/AzureBlobStorageTestAccount.java | 28 +-
.../hadoop/fs/azure/MockStorageInterface.java | 72 ++-
.../fs/azure/NativeAzureFileSystemBaseTest.java | 6 +-
.../TestAzureFileSystemErrorConditions.java | 6 +-
.../hadoop/fs/azure/TestBlobDataValidation.java | 20 +-
.../hadoop/fs/azure/TestContainerChecks.java | 6 +-
.../TestOutOfBandAzureBlobOperationsLive.java | 4 +-
.../fs/azure/TestWasbUriAndConfiguration.java | 4 +-
hadoop-yarn-project/CHANGES.txt | 50 ++
.../hadoop-yarn/bin/yarn-daemons.sh | 13 +-
.../dev-support/findbugs-exclude.xml | 5 +-
.../yarn/api/ApplicationBaseProtocol.java | 355 +++++++++++
.../yarn/api/ApplicationClientProtocol.java | 290 +--------
.../yarn/api/ApplicationHistoryProtocol.java | 303 +--------
.../yarn/api/records/ContainerReport.java | 17 +-
.../yarn/api/records/LogAggregationContext.java | 79 ++-
.../hadoop/yarn/conf/YarnConfiguration.java | 12 +
.../src/main/proto/yarn_protos.proto | 3 +
.../hadoop/yarn/client/cli/ApplicationCLI.java | 11 +-
.../hadoop/yarn/client/ProtocolHATestBase.java | 4 +-
.../yarn/client/api/impl/TestAHSClient.java | 6 +-
.../yarn/client/api/impl/TestYarnClient.java | 16 +-
.../hadoop/yarn/client/cli/TestYarnCLI.java | 18 +-
.../records/impl/pb/ContainerReportPBImpl.java | 19 +
.../impl/pb/LogAggregationContextPBImpl.java | 39 ++
.../client/api/impl/TimelineClientImpl.java | 80 +--
.../logaggregation/AggregatedLogFormat.java | 39 +-
.../util/ResourceCalculatorProcessTree.java | 25 +-
.../hadoop/yarn/util/resource/Resources.java | 5 +
.../apache/hadoop/yarn/webapp/ResponseInfo.java | 6 +-
.../hadoop/yarn/webapp/YarnWebParams.java | 4 +
.../hadoop/yarn/webapp/view/HtmlBlock.java | 2 +
.../src/main/resources/yarn-default.xml | 14 +
.../ApplicationHistoryClientService.java | 185 +++---
.../ApplicationHistoryManager.java | 126 +++-
.../ApplicationHistoryManagerImpl.java | 2 +-
...pplicationHistoryManagerOnTimelineStore.java | 10 +-
.../ApplicationHistoryServer.java | 2 +-
.../webapp/AHSView.java | 28 +-
.../webapp/AHSWebApp.java | 16 +-
.../webapp/AHSWebServices.java | 6 +-
.../webapp/AppAttemptPage.java | 17 +-
.../webapp/AppPage.java | 21 +-
.../TestApplicationHistoryClientService.java | 12 +-
.../webapp/TestAHSWebApp.java | 27 +-
.../webapp/TestAHSWebServices.java | 26 +-
.../TestTimelineAuthenticationFilter.java | 221 +++----
.../yarn/server/api/ApplicationContext.java | 122 ----
.../metrics/ContainerMetricsConstants.java | 2 +
.../yarn/server/webapp/AppAttemptBlock.java | 172 ++++--
.../hadoop/yarn/server/webapp/AppBlock.java | 247 ++++++--
.../hadoop/yarn/server/webapp/AppsBlock.java | 53 +-
.../yarn/server/webapp/ContainerBlock.java | 36 +-
.../hadoop/yarn/server/webapp/WebPageUtils.java | 86 +++
.../hadoop/yarn/server/webapp/WebServices.java | 68 +-
.../hadoop/yarn/server/webapp/dao/AppInfo.java | 11 +-
.../yarn/server/webapp/dao/ContainerInfo.java | 5 +
.../server/nodemanager/ContainerExecutor.java | 49 +-
.../nodemanager/DefaultContainerExecutor.java | 9 +-
.../WindowsSecureContainerExecutor.java | 9 +-
.../logaggregation/AppLogAggregatorImpl.java | 22 +-
.../nodemanager/TestContainerExecutor.java | 53 ++
.../TestContainerManagerRecovery.java | 10 +-
.../TestLogAggregationService.java | 73 ++-
.../metrics/ContainerCreatedEvent.java | 8 +-
.../metrics/SystemMetricsPublisher.java | 5 +-
.../ProportionalCapacityPreemptionPolicy.java | 27 +-
.../rmcontainer/RMContainer.java | 1 +
.../rmcontainer/RMContainerImpl.java | 22 +-
.../resourcemanager/webapp/AppAttemptPage.java | 57 ++
.../server/resourcemanager/webapp/AppBlock.java | 344 -----------
.../server/resourcemanager/webapp/AppPage.java | 25 +-
.../resourcemanager/webapp/AppsBlock.java | 132 ----
.../webapp/AppsBlockWithMetrics.java | 1 +
.../webapp/CapacitySchedulerPage.java | 1 +
.../resourcemanager/webapp/ContainerPage.java | 44 ++
.../webapp/DefaultSchedulerPage.java | 1 +
.../webapp/FairSchedulerPage.java | 21 +-
.../server/resourcemanager/webapp/RMWebApp.java | 5 +
.../resourcemanager/webapp/RmController.java | 8 +
.../server/resourcemanager/webapp/RmView.java | 31 +-
.../resourcemanager/webapp/dao/NodesInfo.java | 3 +
.../webapp/dao/SchedulerTypeInfo.java | 3 +
.../metrics/TestSystemMetricsPublisher.java | 5 +-
...estProportionalCapacityPreemptionPolicy.java | 24 +
.../capacity/TestContainerAllocation.java | 8 +-
.../resourcemanager/webapp/TestAppPage.java | 8 +-
.../resourcemanager/webapp/TestRMWebApp.java | 48 +-
.../webapp/TestRMWebAppFairScheduler.java | 14 +-
.../src/site/markdown/CapacityScheduler.md | 26 +
.../src/site/markdown/ResourceManagerRestart.md | 6 +-
.../src/site/markdown/YarnCommands.md | 28 +-
.../hadoop-yarn/shellprofile.d/yarn | 62 --
.../hadoop-yarn/shellprofile.d/yarn.sh | 62 ++
212 files changed, 5895 insertions(+), 3164 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb1b5960/hadoop-project/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb1b5960/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb1b5960/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb1b5960/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
----------------------------------------------------------------------
diff --cc
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
index d5faaac,df6c7a4..22dcc00
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
@@@ -112,6 -108,9 +113,8 @@@ public class TimelineClientImpl extend
private ConnectionConfigurator connConfigurator;
private DelegationTokenAuthenticator authenticator;
private DelegationTokenAuthenticatedURL.Token token;
- private URI resURI;
+ private UserGroupInformation authUgi;
+ private String doAsUser;
@Private
@VisibleForTesting
@@@ -339,46 -313,20 +351,50 @@@
doPosting(domain, "domain");
}
+ private void putObjects(
+ URI base, String path, MultivaluedMap<String, String> params, Object
obj)
+ throws IOException, YarnException {
+ ClientResponse resp;
+ try {
+ resp = client.resource(base).path(path).queryParams(params)
+ .accept(MediaType.APPLICATION_JSON)
+ .type(MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class, obj);
+ } catch (RuntimeException re) {
+ // runtime exception is expected if the client cannot connect the server
+ String msg =
+ "Failed to get the response from the timeline server.";
+ LOG.error(msg, re);
+ throw new IOException(re);
+ }
+ if (resp == null ||
+ resp.getClientResponseStatus() != ClientResponse.Status.OK) {
+ String msg =
+ "Failed to get the response from the timeline server.";
+ LOG.error(msg);
+ if (LOG.isDebugEnabled() && resp != null) {
+ String output = resp.getEntity(String.class);
+ LOG.debug("HTTP error code: " + resp.getStatus()
+ + " Server response:\n" + output);
+ }
+ throw new YarnException(msg);
+ }
+ }
+
- private ClientResponse doPosting(Object obj, String path) throws
IOException, YarnException {
+ private ClientResponse doPosting(final Object obj, final String path)
+ throws IOException, YarnException {
ClientResponse resp;
try {
- resp = doPostingObject(obj, path);
- } catch (RuntimeException re) {
- // runtime exception is expected if the client cannot connect the server
- String msg =
- "Failed to get the response from the timeline server.";
- LOG.error(msg, re);
- throw re;
+ resp = authUgi.doAs(new PrivilegedExceptionAction<ClientResponse>() {
+ @Override
+ public ClientResponse run() throws Exception {
+ return doPostingObject(obj, path);
+ }
+ });
+ } catch (UndeclaredThrowableException e) {
+ throw new IOException(e.getCause());
+ } catch (InterruptedException ie) {
+ throw new IOException(ie);
}
if (resp == null ||
resp.getClientResponseStatus() != ClientResponse.Status.OK) {
