Author: szetszwo
Date: Wed Feb 26 20:55:50 2014
New Revision: 1572251
URL: http://svn.apache.org/r1572251
Log:
Merge r1569890 through r1572250 from trunk.
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/ (props
changed)
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
(props changed)
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclFeature.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java
Propchange: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1571814-1572250
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1572251&r1=1572250&r2=1572251&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Wed Feb 26 20:55:50 2014
@@ -13,9 +13,6 @@ Trunk (Unreleased)
HDFS-3125. Add JournalService to enable Journal Daemon. (suresh)
- HDFS-4685. Implementation of ACLs in HDFS. (See breakdown of tasks below
for
- features and contributors)
-
IMPROVEMENTS
HDFS-4665. Move TestNetworkTopologyWithNodeGroup to common.
@@ -256,86 +253,6 @@ Trunk (Unreleased)
HDFS-5794. Fix the inconsistency of layout version number of
ADD_DATANODE_AND_STORAGE_UUIDS between trunk and branch-2. (jing9)
- BREAKDOWN OF HDFS-4685 SUBTASKS AND RELATED JIRAS
-
- HDFS-5596. Implement RPC stubs. (Haohui Mai via cnauroth)
-
- HDFS-5685. Implement ACL as a INode feature. (Haohui Mai via cnauroth)
-
- HDFS-5618. NameNode: persist ACLs in fsimage. (Haohui Mai via cnauroth)
-
- HDFS-5619. NameNode: record ACL modifications to edit log.
- (Haohui Mai via cnauroth)
-
- HDFS-5673. Implement logic for modification of ACLs. (cnauroth)
-
- HDFS-5758. NameNode: complete implementation of inode modifications for
- ACLs. (Chris Nauroth via wheat9)
-
- HDFS-5612. NameNode: change all permission checks to enforce ACLs in
- addition to permissions. (Chris Nauroth via wheat9)
-
- HDFS-5613. NameNode: implement handling of ACLs in combination with
- symlinks. (Chris Nauroth via wheat9)
-
- HDFS-5615. NameNode: implement handling of ACLs in combination with sticky
- bit. (Chris Nauroth via wheat9)
-
- HDFS-5702. FsShell Cli: Add XML based End-to-End test for getfacl and
- setfacl commands. (Vinay via cnauroth)
-
- HDFS-5608. WebHDFS: implement ACL APIs.
- (Sachin Jose and Renil Joseph via cnauroth)
-
- HDFS-5614. NameNode: implement handling of ACLs in combination with
- snapshots. (cnauroth)
-
- HDFS-5858. Refactor common ACL test cases to be run through multiple
- FileSystem implementations. (cnauroth)
-
- HDFS-5860. Refactor INodeDirectory getDirectoryXFeature methods to use
- common getFeature helper method. (Jing Zhao via cnauroth)
-
- HDFS-5861. Add CLI test for Ls output for extended ACL marker.
- (Vinay via cnauroth)
-
- HDFS-5616. NameNode: implement default ACL handling. (cnauroth)
-
- HDFS-5899. Add configuration flag to disable/enable support for ACLs.
- (cnauroth)
-
- HDFS-5914. Incorporate ACLs with the changes from HDFS-5698.
- (Haohui Mai via cnauroth)
-
- HDFS-5625. Write end user documentation for HDFS ACLs. (cnauroth)
-
- HDFS-5925. ACL configuration flag must only reject ACL API calls, not ACLs
- present in fsimage or edits. (cnauroth)
-
- HDFS-5923. Do not persist the ACL bit in the FsPermission.
- (Haohui Mai via cnauroth)
-
- HDFS-5933. Optimize the FSImage layout for ACLs (Haohui Mai via cnauroth)
-
- HDFS-5932. Ls should display the ACL bit (Chris Nauroth via wheat9)
-
- HDFS-5937. Fix TestOfflineEditsViewer on HDFS-4685 branch. (cnauroth)
-
- HDFS-5737. Replacing only the default ACL can fail to copy unspecified base
- entries from the access ACL. (cnauroth)
-
- HDFS-5739. ACL RPC must allow null name or unspecified permissions in ACL
- entries. (cnauroth)
-
- HDFS-5799. Make audit logging consistent across ACL APIs. (cnauroth)
-
- HDFS-5849. Removing ACL from an inode fails if it has only a default ACL.
- (cnauroth)
-
- HDFS-5623. NameNode: add tests for skipping ACL enforcement when permission
- checks are disabled, user is superuser or user is member of supergroup.
- (cnauroth)
-
Release 2.5.0 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -359,6 +276,9 @@ Release 2.4.0 - UNRELEASED
HDFS-5776 Support 'hedged' reads in DFSClient (Liang Xie via stack)
+ HDFS-4685. Implementation of ACLs in HDFS. (See breakdown of tasks below
for
+ features and contributors)
+
IMPROVEMENTS
HDFS-5781. Use an array to record the mapping between FSEditLogOpCode and
@@ -439,6 +359,9 @@ Release 2.4.0 - UNRELEASED
HDFS-6006. Remove duplicate code in FSNameSystem#getFileInfo.
(Akira Ajisaka via cnauroth)
+ HDFS-6018. Exception recorded in LOG when IPCLoggerChannel#close is called.
+ (jing9)
+
OPTIMIZATIONS
HDFS-5790. LeaseManager.findPath is very slow when many leases need
recovery
@@ -563,6 +486,14 @@ Release 2.4.0 - UNRELEASED
HDFS-5988. Bad fsimage always generated after upgrade. (wang)
+ HDFS-5922. DN heartbeat thread can get stuck in tight loop. (Arpit Agarwal)
+
+ HDFS-6008. Namenode dead node link is giving HTTP error 500.
+ (Benoy Antony via cnauroth)
+
+ HDFS-5936. MiniDFSCluster does not clean data left behind by
+ SecondaryNameNode. (Binglin Chang via cnauroth)
+
BREAKDOWN OF HDFS-5698 SUBTASKS AND RELATED JIRAS
HDFS-5717. Save FSImage header in protobuf. (Haohui Mai via jing9)
@@ -623,7 +554,88 @@ Release 2.4.0 - UNRELEASED
HDFS-5981. PBImageXmlWriter generates malformed XML.
(Haohui Mai via cnauroth)
- HDFS-5922. DN heartbeat thread can get stuck in tight loop. (Arpit Agarwal)
+ BREAKDOWN OF HDFS-4685 SUBTASKS AND RELATED JIRAS
+
+ HDFS-5596. Implement RPC stubs. (Haohui Mai via cnauroth)
+
+ HDFS-5685. Implement ACL as a INode feature. (Haohui Mai via cnauroth)
+
+ HDFS-5618. NameNode: persist ACLs in fsimage. (Haohui Mai via cnauroth)
+
+ HDFS-5619. NameNode: record ACL modifications to edit log.
+ (Haohui Mai via cnauroth)
+
+ HDFS-5673. Implement logic for modification of ACLs. (cnauroth)
+
+ HDFS-5758. NameNode: complete implementation of inode modifications for
+ ACLs. (Chris Nauroth via wheat9)
+
+ HDFS-5612. NameNode: change all permission checks to enforce ACLs in
+ addition to permissions. (Chris Nauroth via wheat9)
+
+ HDFS-5613. NameNode: implement handling of ACLs in combination with
+ symlinks. (Chris Nauroth via wheat9)
+
+ HDFS-5615. NameNode: implement handling of ACLs in combination with sticky
+ bit. (Chris Nauroth via wheat9)
+
+ HDFS-5702. FsShell Cli: Add XML based End-to-End test for getfacl and
+ setfacl commands. (Vinay via cnauroth)
+
+ HDFS-5608. WebHDFS: implement ACL APIs.
+ (Sachin Jose and Renil Joseph via cnauroth)
+
+ HDFS-5614. NameNode: implement handling of ACLs in combination with
+ snapshots. (cnauroth)
+
+ HDFS-5858. Refactor common ACL test cases to be run through multiple
+ FileSystem implementations. (cnauroth)
+
+ HDFS-5860. Refactor INodeDirectory getDirectoryXFeature methods to use
+ common getFeature helper method. (Jing Zhao via cnauroth)
+
+ HDFS-5861. Add CLI test for Ls output for extended ACL marker.
+ (Vinay via cnauroth)
+
+ HDFS-5616. NameNode: implement default ACL handling. (cnauroth)
+
+ HDFS-5899. Add configuration flag to disable/enable support for ACLs.
+ (cnauroth)
+
+ HDFS-5914. Incorporate ACLs with the changes from HDFS-5698.
+ (Haohui Mai via cnauroth)
+
+ HDFS-5625. Write end user documentation for HDFS ACLs. (cnauroth)
+
+ HDFS-5925. ACL configuration flag must only reject ACL API calls, not ACLs
+ present in fsimage or edits. (cnauroth)
+
+ HDFS-5923. Do not persist the ACL bit in the FsPermission.
+ (Haohui Mai via cnauroth)
+
+ HDFS-5933. Optimize the FSImage layout for ACLs (Haohui Mai via cnauroth)
+
+ HDFS-5932. Ls should display the ACL bit (Chris Nauroth via wheat9)
+
+ HDFS-5937. Fix TestOfflineEditsViewer on HDFS-4685 branch. (cnauroth)
+
+ HDFS-5737. Replacing only the default ACL can fail to copy unspecified base
+ entries from the access ACL. (cnauroth)
+
+ HDFS-5739. ACL RPC must allow null name or unspecified permissions in ACL
+ entries. (cnauroth)
+
+ HDFS-5799. Make audit logging consistent across ACL APIs. (cnauroth)
+
+ HDFS-5849. Removing ACL from an inode fails if it has only a default ACL.
+ (cnauroth)
+
+ HDFS-5623. NameNode: add tests for skipping ACL enforcement when permission
+ checks are disabled, user is superuser or user is member of supergroup.
+ (cnauroth)
+
+ HDFS-5908. Change AclFeature to capture list of ACL entries in an
+ ImmutableList. (cnauroth)
Release 2.3.1 - UNRELEASED
Propchange:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1571814-1572250
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java?rev=1572251&r1=1572250&r2=1572251&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java
Wed Feb 26 20:55:50 2014
@@ -182,7 +182,6 @@ public class IPCLoggerChannel implements
@Override
public void close() {
- QuorumJournalManager.LOG.info("Closing", new Exception());
// No more tasks may be submitted after this point.
executor.shutdown();
if (proxy != null) {
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java?rev=1572251&r1=1572250&r2=1572251&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java
Wed Feb 26 20:55:50 2014
@@ -147,7 +147,9 @@ public class JspHelper {
*/
public static final class Url {
public static String authority(String scheme, DatanodeID d) {
- String fqdn = canonicalize(d.getIpAddr());
+ String fqdn = (d.getIpAddr() != null && !d.getIpAddr().isEmpty())?
+ canonicalize(d.getIpAddr()):
+ d.getHostName();
if (scheme.equals("http")) {
return fqdn + ":" + d.getInfoPort();
} else if (scheme.equals("https")) {
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclFeature.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclFeature.java?rev=1572251&r1=1572250&r2=1572251&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclFeature.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclFeature.java
Wed Feb 26 20:55:50 2014
@@ -18,26 +18,26 @@
package org.apache.hadoop.hdfs.server.namenode;
-import java.util.Collections;
-import java.util.List;
-
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.permission.AclEntry;
+import com.google.common.collect.ImmutableList;
+
/**
* Feature that represents the ACLs of the inode.
*/
@InterfaceAudience.Private
public class AclFeature implements INode.Feature {
- public static final List<AclEntry> EMPTY_ENTRY_LIST =
Collections.emptyList();
+ public static final ImmutableList<AclEntry> EMPTY_ENTRY_LIST =
+ ImmutableList.of();
- private final List<AclEntry> entries;
+ private final ImmutableList<AclEntry> entries;
- public AclFeature(List<AclEntry> entries) {
+ public AclFeature(ImmutableList<AclEntry> entries) {
this.entries = entries;
}
- public List<AclEntry> getEntries() {
+ public ImmutableList<AclEntry> getEntries() {
return entries;
}
}
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java?rev=1572251&r1=1572250&r2=1572251&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java
Wed Feb 26 20:55:50 2014
@@ -328,7 +328,7 @@ final class AclStorage {
// Add all default entries to the feature.
featureEntries.addAll(defaultEntries);
- return new AclFeature(Collections.unmodifiableList(featureEntries));
+ return new AclFeature(ImmutableList.copyOf(featureEntries));
}
/**
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java?rev=1572251&r1=1572250&r2=1572251&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
Wed Feb 26 20:55:50 2014
@@ -79,6 +79,7 @@ import org.apache.hadoop.hdfs.protocol.E
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.hdfs.server.common.Storage;
+import org.apache.hadoop.hdfs.server.common.Util;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
import org.apache.hadoop.hdfs.server.datanode.DataStorage;
@@ -817,6 +818,14 @@ public class MiniDFSCluster {
throw new IOException("Could not fully delete " + nameDir);
}
}
+ Collection<URI> checkpointDirs = Util.stringCollectionAsURIs(conf
+ .getTrimmedStringCollection(DFS_NAMENODE_CHECKPOINT_DIR_KEY));
+ for (URI checkpointDirUri : checkpointDirs) {
+ File checkpointDir = new File(checkpointDirUri);
+ if (checkpointDir.exists() &&
!FileUtil.fullyDelete(checkpointDir)) {
+ throw new IOException("Could not fully delete " + checkpointDir);
+ }
+ }
}
boolean formatThisOne = format;
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java?rev=1572251&r1=1572250&r2=1572251&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java
Wed Feb 26 20:55:50 2014
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.co
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
@@ -641,5 +642,20 @@ public class TestJspHelper {
assertTrue(upgradeStatusReport.getStatusText(true).equals(
MessageFormat.format(EXPECTED__NOTF_PATTERN, version)));
}
+
+ @Test
+ public void testAuthority(){
+ DatanodeID dnWithIp = new DatanodeID("127.0.0.1", "hostName", null,
+ 50020, 50075, 50076, 50010);
+ assertNotNull(JspHelper.Url.authority("http", dnWithIp));
+
+ DatanodeID dnWithNullIp = new DatanodeID(null, "hostName", null,
+ 50020, 50075, 50076, 50010);
+ assertNotNull(JspHelper.Url.authority("http", dnWithNullIp));
+
+ DatanodeID dnWithEmptyIp = new DatanodeID("", "hostName", null,
+ 50020, 50075, 50076, 50010);
+ assertNotNull(JspHelper.Url.authority("http", dnWithEmptyIp));
+ }
}
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java?rev=1572251&r1=1572250&r2=1572251&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java
Wed Feb 26 20:55:50 2014
@@ -48,6 +48,7 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
/**
@@ -1272,6 +1273,12 @@ public abstract class FSAclBaseTest {
AclFeature aclFeature = inode.getAclFeature();
if (expectAclFeature) {
assertNotNull(aclFeature);
+ // Intentionally capturing a reference to the entries, not using nested
+ // calls. This way, we get compile-time enforcement that the entries are
+ // stored in an ImmutableList.
+ ImmutableList<AclEntry> entries = aclFeature.getEntries();
+ assertNotNull(entries);
+ assertFalse(entries.isEmpty());
} else {
assertNull(aclFeature);
}
