Merge branch 'trunk' into HDFS-6581
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e0d7fb48
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e0d7fb48
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e0d7fb48
Branch: refs/heads/HDFS-6581
Commit: e0d7fb48fbe872ef47803ceecdae654ab8c9d8af
Parents: a7814e1 ffdb7eb
Author: arp <a...@apache.org>
Authored: Tue Sep 16 14:32:57 2014 -0700
Committer: arp <a...@apache.org>
Committed: Tue Sep 16 14:32:57 2014 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-auth/pom.xml | 13 +
.../server/AuthenticationFilter.java | 152 ++++--
.../util/RandomSignerSecretProvider.java | 4 +-
.../util/RolloverSignerSecretProvider.java | 7 +-
.../util/SignerSecretProvider.java | 9 +-
.../util/StringSignerSecretProvider.java | 15 +-
.../util/ZKSignerSecretProvider.java | 506 +++++++++++++++++++
.../src/site/apt/Configuration.apt.vm | 148 +++++-
.../hadoop-auth/src/site/apt/index.apt.vm | 5 +
.../server/TestAuthenticationFilter.java | 117 ++++-
.../util/TestJaasConfiguration.java | 55 ++
.../util/TestRandomSignerSecretProvider.java | 2 +-
.../util/TestRolloverSignerSecretProvider.java | 2 +-
.../authentication/util/TestSigner.java | 23 +-
.../util/TestStringSignerSecretProvider.java | 9 +-
.../util/TestZKSignerSecretProvider.java | 270 ++++++++++
hadoop-common-project/hadoop-common/CHANGES.txt | 8 +
.../src/main/bin/hadoop-functions.sh | 118 +++--
.../hadoop/fs/http/server/TestHttpFSServer.java | 8 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 6 +
.../org/apache/hadoop/hdfs/DFSOutputStream.java | 40 +-
.../datatransfer/DataTransferProtoUtil.java | 44 +-
.../hdfs/protocol/datatransfer/Receiver.java | 125 +++--
.../hdfs/protocol/datatransfer/Sender.java | 26 +-
.../BlockPlacementPolicyDefault.java | 2 +-
.../src/main/proto/datatransfer.proto | 8 +
.../blockmanagement/TestBlockManager.java | 55 ++
.../org/apache/hadoop/tracing/TestTracing.java | 52 +-
.../TestTracingShortCircuitLocalRead.java | 97 ++++
hadoop-project/pom.xml | 11 +
.../org/apache/hadoop/fs/s3a/Constants.java | 36 +-
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 46 +-
.../apache/hadoop/fs/s3a/S3AOutputStream.java | 6 +-
hadoop-yarn-project/CHANGES.txt | 7 +
.../hadoop/yarn/conf/YarnConfiguration.java | 10 +
.../applications/distributedshell/Client.java | 18 +-
.../TestDSSleepingAppMaster.java | 58 +++
.../distributedshell/TestDistributedShell.java | 76 +++
.../src/main/resources/yarn-default.xml | 10 +
.../util/CgroupsLCEResourcesHandler.java | 29 +-
.../util/TestCgroupsLCEResourcesHandler.java | 165 +++++-
41 files changed, 2117 insertions(+), 281 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0d7fb48/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
----------------------------------------------------------------------
diff --cc
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
index ed1abd8,d368f4e..6ff306c
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
@@@ -359,10 -364,17 +365,18 @@@ public class DFSOutputStream extends FS
/**
* Default construction for file create
*/
- private DataStreamer() {
- this(null);
+ private DataStreamer(HdfsFileStatus stat) {
++ this(stat, null);
+ }
+
+ /**
+ * construction with tracing info
+ */
- private DataStreamer(Span span) {
++ private DataStreamer(HdfsFileStatus stat, Span span) {
isAppend = false;
+ isLazyPersistFile = stat.isLazyPersist();
stage = BlockConstructionStage.PIPELINE_SETUP_CREATE;
+ traceSpan = span;
}
/**
@@@ -1614,7 -1633,11 +1636,11 @@@
computePacketChunkSize(dfsClient.getConf().writePacketSize,
checksum.getBytesPerChecksum());
- streamer = new DataStreamer(stat);
+ Span traceSpan = null;
+ if (Trace.isTracing()) {
+ traceSpan = Trace.startSpan(this.getClass().getSimpleName()).detach();
+ }
- streamer = new DataStreamer(traceSpan);
++ streamer = new DataStreamer(stat, traceSpan);
if (favoredNodes != null && favoredNodes.length != 0) {
streamer.setFavoredNodes(favoredNodes);
}
@@@ -1663,7 -1692,7 +1695,7 @@@
} else {
computePacketChunkSize(dfsClient.getConf().writePacketSize,
checksum.getBytesPerChecksum());
- streamer = new DataStreamer(stat);
- streamer = new DataStreamer(traceSpan);
++ streamer = new DataStreamer(stat, traceSpan);
}
this.fileEncryptionInfo = stat.getFileEncryptionInfo();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0d7fb48/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java
----------------------------------------------------------------------
diff --cc
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java
index 78693bb,daae9b7..8192925
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java
@@@ -123,22 -131,27 +131,28 @@@ public abstract class Receiver implemen
private void opWriteBlock(DataInputStream in) throws IOException {
final OpWriteBlockProto proto =
OpWriteBlockProto.parseFrom(vintPrefixed(in));
final DatanodeInfo[] targets = PBHelper.convert(proto.getTargetsList());
- writeBlock(PBHelper.convert(proto.getHeader().getBaseHeader().getBlock()),
- PBHelper.convertStorageType(proto.getStorageType()),
- PBHelper.convert(proto.getHeader().getBaseHeader().getToken()),
- proto.getHeader().getClientName(),
- targets,
- PBHelper.convertStorageTypes(proto.getTargetStorageTypesList(),
targets.length),
- PBHelper.convert(proto.getSource()),
- fromProto(proto.getStage()),
- proto.getPipelineSize(),
- proto.getMinBytesRcvd(), proto.getMaxBytesRcvd(),
- proto.getLatestGenerationStamp(),
- fromProto(proto.getRequestedChecksum()),
- (proto.hasCachingStrategy() ?
- getCachingStrategy(proto.getCachingStrategy()) :
- CachingStrategy.newDefaultStrategy()),
- (proto.hasAllowLazyPersist() ? proto.getAllowLazyPersist() : false));
+ TraceScope traceScope = continueTraceSpan(proto.getHeader(),
+ proto.getClass().getSimpleName());
+ try {
+
writeBlock(PBHelper.convert(proto.getHeader().getBaseHeader().getBlock()),
+ PBHelper.convertStorageType(proto.getStorageType()),
+ PBHelper.convert(proto.getHeader().getBaseHeader().getToken()),
+ proto.getHeader().getClientName(),
+ targets,
+ PBHelper.convertStorageTypes(proto.getTargetStorageTypesList(),
targets.length),
+ PBHelper.convert(proto.getSource()),
+ fromProto(proto.getStage()),
+ proto.getPipelineSize(),
+ proto.getMinBytesRcvd(), proto.getMaxBytesRcvd(),
+ proto.getLatestGenerationStamp(),
+ fromProto(proto.getRequestedChecksum()),
+ (proto.hasCachingStrategy() ?
+ getCachingStrategy(proto.getCachingStrategy()) :
- CachingStrategy.newDefaultStrategy()));
- } finally {
- if (traceScope != null) traceScope.close();
- }
++ CachingStrategy.newDefaultStrategy()),
++ (proto.hasAllowLazyPersist() ? proto.getAllowLazyPersist() :
false));
++ } finally {
++ if (traceScope != null) traceScope.close();
++ }
}
/** Receive {@link Op#TRANSFER_BLOCK} */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0d7fb48/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Sender.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0d7fb48/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto
----------------------------------------------------------------------
