Merge trunk into HDFS-6581
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/339d21f2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/339d21f2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/339d21f2
Branch: refs/heads/HDFS-6581
Commit: 339d21f273a7c32be989c2cc2b3b1f0dd8a2ec0f
Parents: cb9b485 e6420fe
Author: arp <[email protected]>
Authored: Fri Sep 5 21:36:25 2014 -0700
Committer: arp <[email protected]>
Committed: Fri Sep 5 21:36:25 2014 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 25 +++
.../dev-support/findbugsExcludeFile.xml | 2 +-
.../src/main/bin/hadoop-functions.sh | 4 +-
.../hadoop/crypto/key/KeyProviderFactory.java | 36 +++-
.../crypto/key/kms/KMSClientProvider.java | 57 +-----
.../hadoop/crypto/random/OsSecureRandom.java | 21 ++-
.../security/authorize/AccessControlList.java | 12 +-
.../security/ssl/FileBasedKeyStoresFactory.java | 2 +-
.../DelegationTokenAuthenticationFilter.java | 15 +-
.../DelegationTokenAuthenticationHandler.java | 6 +-
.../web/DelegationTokenAuthenticator.java | 20 +-
.../apache/hadoop/util/HttpExceptionUtils.java | 185 +++++++++++++++++++
.../apache/hadoop/crypto/TestCryptoCodec.java | 69 +++++--
.../apache/hadoop/crypto/TestCryptoStreams.java | 2 +-
.../crypto/key/TestKeyProviderFactory.java | 13 ++
.../crypto/random/TestOsSecureRandom.java | 15 ++
...tionTokenAuthenticationHandlerWithMocks.java | 35 ++--
.../hadoop/util/TestHttpExceptionUtils.java | 167 +++++++++++++++++
hadoop-common-project/hadoop-kms/pom.xml | 1 +
.../hadoop/crypto/key/kms/server/KMS.java | 27 +--
.../hadoop/crypto/key/kms/server/KMSACLs.java | 55 +++++-
.../key/kms/server/KMSExceptionsProvider.java | 12 +-
.../hadoop-kms/src/site/apt/index.apt.vm | 88 ++++++++-
.../hadoop/crypto/key/kms/server/TestKMS.java | 100 +++++++++-
.../crypto/key/kms/server/TestKMSACLs.java | 2 +-
.../hadoop/fs/http/client/HttpFSFileSystem.java | 70 ++++---
.../hadoop/fs/http/client/HttpFSUtils.java | 50 -----
.../hadoop/lib/wsrs/ExceptionProvider.java | 14 +-
.../fs/http/client/BaseTestHttpFSWith.java | 4 +-
.../fs/http/server/TestHttpFSServerNoACLs.java | 10 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 28 +++
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 6 +-
.../hadoop-hdfs/src/main/bin/hdfs | 15 +-
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 3 +
.../java/org/apache/hadoop/hdfs/DFSUtil.java | 80 +++++++-
.../hadoop/hdfs/DistributedFileSystem.java | 11 +-
.../org/apache/hadoop/hdfs/inotify/Event.java | 12 ++
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 6 +-
.../hdfs/server/datanode/BlockPoolManager.java | 12 +-
.../hdfs/server/datanode/DataStorage.java | 2 +-
.../hdfs/server/namenode/FSDirectory.java | 5 +-
.../hadoop/hdfs/server/namenode/FSEditLog.java | 6 +-
.../hdfs/server/namenode/FSEditLogLoader.java | 8 +-
.../hdfs/server/namenode/FSEditLogOp.java | 18 ++
.../server/namenode/FSImageSerialization.java | 17 ++
.../hdfs/server/namenode/FSNamesystem.java | 46 +++--
.../hdfs/server/namenode/INodeReference.java | 4 +-
.../namenode/InotifyFSEditLogOpTranslator.java | 1 +
.../server/namenode/NameNodeLayoutVersion.java | 5 +-
.../snapshot/DirectorySnapshottableFeature.java | 2 +-
.../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 175 ++++++++----------
.../org/apache/hadoop/hdfs/tools/GetConf.java | 4 +-
.../hadoop-hdfs/src/main/proto/inotify.proto | 1 +
.../src/main/resources/hdfs-default.xml | 18 ++
.../hadoop/hdfs/TestBlocksScheduledCounter.java | 18 +-
.../hdfs/TestDFSInotifyEventInputStream.java | 8 +-
.../org/apache/hadoop/hdfs/TestDFSUtil.java | 26 +++
.../apache/hadoop/hdfs/TestFileCreation.java | 119 ++++++++++++
.../apache/hadoop/hdfs/TestHDFSServerPorts.java | 24 +--
.../org/apache/hadoop/hdfs/TestLocalDFS.java | 29 ++-
.../server/datanode/TestBlockPoolManager.java | 22 +++
.../hdfs/server/namenode/CreateEditsLog.java | 2 +-
.../hdfs/server/namenode/TestEditLog.java | 2 +-
.../TestValidateConfigurationSettings.java | 16 +-
.../namenode/ha/TestDelegationTokensWithHA.java | 63 +++----
.../hdfs/server/namenode/ha/TestHAMetrics.java | 10 +-
.../namenode/ha/TestHAStateTransitions.java | 52 +++---
.../namenode/ha/TestStandbyCheckpoints.java | 54 +++---
.../snapshot/TestSnapshotDiffReport.java | 36 ++++
.../apache/hadoop/hdfs/tools/TestGetConf.java | 26 ++-
.../java/org/apache/hadoop/tools/TestTools.java | 2 +-
.../hadoop-hdfs/src/test/resources/editsStored | Bin 4992 -> 5252 bytes
.../src/test/resources/editsStored.xml | 182 +++++++++++-------
hadoop-mapreduce-project/CHANGES.txt | 6 +
.../mapreduce/v2/app/job/impl/JobImpl.java | 2 +
.../java/org/apache/hadoop/mapred/MapTask.java | 4 +-
hadoop-yarn-project/CHANGES.txt | 12 ++
.../hadoop/yarn/conf/YarnConfiguration.java | 8 +
.../ApplicationHistoryServer.java | 19 +-
.../timeline/webapp/CrossOriginFilter.java | 10 +-
.../webapp/CrossOriginFilterInitializer.java | 12 +-
.../timeline/webapp/TestCrossOriginFilter.java | 15 ++
.../TestCrossOriginFilterInitializer.java | 7 +-
.../nodemanager/LinuxContainerExecutor.java | 11 ++
.../nodemanager/TestLinuxContainerExecutor.java | 60 ++++++
pom.xml | 1 +
86 files changed, 1786 insertions(+), 676 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/339d21f2/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/339d21f2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/339d21f2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/339d21f2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/339d21f2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/339d21f2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/339d21f2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/339d21f2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
----------------------------------------------------------------------
diff --cc
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
index 94c287c,c5dd3df..a9f7f93
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
@@@ -667,10 -666,10 +682,12 @@@ public abstract class FSEditLogOp
Long.toString(atime));
XMLUtils.addSaxString(contentHandler, "BLOCKSIZE",
Long.toString(blockSize));
+ XMLUtils.addSaxString(contentHandler, "LAZY_PERSIST",
+ Boolean.toString(isLazyPersist));
XMLUtils.addSaxString(contentHandler, "CLIENT_NAME", clientName);
XMLUtils.addSaxString(contentHandler, "CLIENT_MACHINE", clientMachine);
+ XMLUtils.addSaxString(contentHandler, "OVERWRITE",
+ Boolean.toString(overwrite));
for (Block b : blocks) {
FSEditLogOp.blockToXml(contentHandler, b);
}
@@@ -692,13 -691,9 +709,14 @@@
this.mtime = Long.parseLong(st.getValue("MTIME"));
this.atime = Long.parseLong(st.getValue("ATIME"));
this.blockSize = Long.parseLong(st.getValue("BLOCKSIZE"));
+
+ String lazyPersistString = st.getValueOrNull("LAZY_PERSIST");
+ this.isLazyPersist =
+ lazyPersistString != null &&
Boolean.parseBoolean(lazyPersistString);
+
this.clientName = st.getValue("CLIENT_NAME");
this.clientMachine = st.getValue("CLIENT_MACHINE");
+ this.overwrite = Boolean.parseBoolean(st.getValueOrNull("OVERWRITE"));
if (st.hasChildren("BLOCK")) {
List<Stanza> blocks = st.getChildren("BLOCK");
this.blocks = new Block[blocks.size()];
http://git-wip-us.apache.org/repos/asf/hadoop/blob/339d21f2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/339d21f2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --cc
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 6116ef1,c1744f6..af8e3ad
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@@ -2529,9 -2497,9 +2530,9 @@@ public class FSNamesystem implements Na
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot create file" + src);
src = resolvePath(src, pathComponents);
- startFileInternal(pc, src, permissions, holder, clientMachine,
create,
- overwrite, createParent, replication, blockSize, isLazyPersist,
- suite, edek, logRetryCache);
+ toRemoveBlocks = startFileInternal(pc, src, permissions, holder,
+ clientMachine, create, overwrite, createParent, replication,
- blockSize, suite, edek, logRetryCache);
++ blockSize, isLazyPersist, suite, edek, logRetryCache);
stat = dir.getFileInfo(src, false,
FSDirectory.isReservedRawName(srcArg));
} catch (StandbyException se) {
@@@ -2568,11 -2540,11 +2573,12 @@@
* For description of parameters and exceptions thrown see
* {@link ClientProtocol#create}
*/
- private void startFileInternal(FSPermissionChecker pc, String src,
- PermissionStatus permissions, String holder, String clientMachine,
- boolean create, boolean overwrite, boolean createParent,
- short replication, long blockSize, boolean isLazyPersist,
- CipherSuite suite, EncryptedKeyVersion edek, boolean logRetryEntry)
+ private BlocksMapUpdateInfo startFileInternal(FSPermissionChecker pc,
+ String src, PermissionStatus permissions, String holder,
+ String clientMachine, boolean create, boolean overwrite,
+ boolean createParent, short replication, long blockSize,
- CipherSuite suite, EncryptedKeyVersion edek, boolean logRetryEntry)
++ boolean isLazyPersist, CipherSuite suite,
++ EncryptedKeyVersion edek, boolean logRetryEntry)
throws FileAlreadyExistsException, AccessControlException,
UnresolvedLinkException, FileNotFoundException,
ParentNotDirectoryException, RetryStartFileException, IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/339d21f2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java
----------------------------------------------------------------------
diff --cc
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java
index 16d55fd,1df6df4..9d7ee9d
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java
@@@ -66,8 -66,8 +66,11 @@@ public class NameNodeLayoutVersion
ROLLING_UPGRADE(-55, -53, "Support rolling upgrade", false),
EDITLOG_LENGTH(-56, "Add length field to every edit log op"),
XATTRS(-57, "Extended attributes"),
- LAZY_PERSIST_FILES(-58, "Support for optional lazy persistence of "
+ CREATE_OVERWRITE(-58, "Use single editlog record for " +
- "creating file with overwrite");
++ "creating file with overwrite"),
++ LAZY_PERSIST_FILES(-59, "Support for optional lazy persistence of "
+ + " files with reduced durability guarantees");
++
private final FeatureInfo info;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/339d21f2/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/339d21f2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
----------------------------------------------------------------------
diff --cc
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
index 7f193f5,3f96c0c..1b5e4a0
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
@@@ -97,9 -97,9 +97,9 @@@ public class CreateEditsLog
editLog.logMkDir(currentDir, dirInode);
}
INodeFile fileUc = new INodeFile(inodeId.nextValue(), null,
- p, 0L, 0L, BlockInfo.EMPTY_ARRAY, replication, blockSize);
+ p, 0L, 0L, BlockInfo.EMPTY_ARRAY, replication, blockSize, false);
fileUc.toUnderConstruction("", "");
- editLog.logOpenFile(filePath, fileUc, false);
+ editLog.logOpenFile(filePath, fileUc, false, false);
editLog.logCloseFile(filePath, inode);
if (currentBlockId - bidAtSync >= 2000) { // sync every 2K blocks
http://git-wip-us.apache.org/repos/asf/hadoop/blob/339d21f2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
----------------------------------------------------------------------
diff --cc
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
index 6c1e959,7b62242..d6af5f2
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
@@@ -194,10 -194,10 +194,10 @@@ public class TestEditLog
for (int i = 0; i < numTransactions; i++) {
INodeFile inode = new INodeFile(namesystem.allocateNewInodeId(), null,
- p, 0L, 0L, BlockInfo.EMPTY_ARRAY, replication, blockSize);
+ p, 0L, 0L, BlockInfo.EMPTY_ARRAY, replication, blockSize, false);
inode.toUnderConstruction("", "");
- editLog.logOpenFile("/filename" + (startIndex + i), inode, false);
+ editLog.logOpenFile("/filename" + (startIndex + i), inode, false,
false);
editLog.logCloseFile("/filename" + (startIndex + i), inode);
editLog.logSync();
}
