Author: cnauroth
Date: Fri May 16 18:25:23 2014
New Revision: 1595283
URL: http://svn.apache.org/r1595283
Log:
MAPREDUCE-5809. Enhance distcp to support preserving HDFS ACLs. Contributed by
Chris Nauroth.
Added:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclUtil.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/ScopedAclEntries.java
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java?rev=1595283&r1=1595282&r2=1595283&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
Fri May 16 18:25:23 2014
@@ -100,6 +100,21 @@ public class FileStatus implements Writa
}
/**
+ * Copy constructor.
+ *
+ * @param other FileStatus to copy
+ */
+ public FileStatus(FileStatus other) throws IOException {
+ // It's important to call the getters here instead of directly accessing
the
+ // members. Subclasses like ViewFsFileStatus can override the getters.
+ this(other.getLen(), other.isDirectory(), other.getReplication(),
+ other.getBlockSize(), other.getModificationTime(), other.getAccessTime(),
+ other.getPermission(), other.getOwner(), other.getGroup(),
+ (other.isSymlink() ? other.getSymlink() : null),
+ other.getPath());
+ }
+
+ /**
* Get the length of this file, in bytes.
* @return the length of this file, in bytes.
*/
Added:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclUtil.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclUtil.java?rev=1595283&view=auto
==============================================================================
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclUtil.java
(added)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclUtil.java
Fri May 16 18:25:23 2014
@@ -0,0 +1,134 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.permission;
+
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import com.google.common.collect.Lists;
+
+/**
+ * AclUtil contains utility methods for manipulating ACLs.
+ */
+@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
+@InterfaceStability.Unstable
+public final class AclUtil {
+
+ /**
+ * Given permissions and extended ACL entries, returns the full logical ACL.
+ *
+ * @param perm FsPermission containing permissions
+ * @param entries List<AclEntry> containing extended ACL entries
+ * @return List<AclEntry> containing full logical ACL
+ */
+ public static List<AclEntry> getAclFromPermAndEntries(FsPermission perm,
+ List<AclEntry> entries) {
+ List<AclEntry> acl = Lists.newArrayListWithCapacity(entries.size() + 3);
+
+ // Owner entry implied by owner permission bits.
+ acl.add(new AclEntry.Builder()
+ .setScope(AclEntryScope.ACCESS)
+ .setType(AclEntryType.USER)
+ .setPermission(perm.getUserAction())
+ .build());
+
+ // All extended access ACL entries.
+ boolean hasAccessAcl = false;
+ Iterator<AclEntry> entryIter = entries.iterator();
+ AclEntry curEntry = null;
+ while (entryIter.hasNext()) {
+ curEntry = entryIter.next();
+ if (curEntry.getScope() == AclEntryScope.DEFAULT) {
+ break;
+ }
+ hasAccessAcl = true;
+ acl.add(curEntry);
+ }
+
+ // Mask entry implied by group permission bits, or group entry if there is
+ // no access ACL (only default ACL).
+ acl.add(new AclEntry.Builder()
+ .setScope(AclEntryScope.ACCESS)
+ .setType(hasAccessAcl ? AclEntryType.MASK : AclEntryType.GROUP)
+ .setPermission(perm.getGroupAction())
+ .build());
+
+ // Other entry implied by other bits.
+ acl.add(new AclEntry.Builder()
+ .setScope(AclEntryScope.ACCESS)
+ .setType(AclEntryType.OTHER)
+ .setPermission(perm.getOtherAction())
+ .build());
+
+ // Default ACL entries.
+ if (curEntry != null && curEntry.getScope() == AclEntryScope.DEFAULT) {
+ acl.add(curEntry);
+ while (entryIter.hasNext()) {
+ acl.add(entryIter.next());
+ }
+ }
+
+ return acl;
+ }
+
+ /**
+ * Translates the given permission bits to the equivalent minimal ACL.
+ *
+ * @param perm FsPermission to translate
+ * @return List<AclEntry> containing exactly 3 entries representing the
owner,
+ * group and other permissions
+ */
+ public static List<AclEntry> getMinimalAcl(FsPermission perm) {
+ return Lists.newArrayList(
+ new AclEntry.Builder()
+ .setScope(AclEntryScope.ACCESS)
+ .setType(AclEntryType.USER)
+ .setPermission(perm.getUserAction())
+ .build(),
+ new AclEntry.Builder()
+ .setScope(AclEntryScope.ACCESS)
+ .setType(AclEntryType.GROUP)
+ .setPermission(perm.getGroupAction())
+ .build(),
+ new AclEntry.Builder()
+ .setScope(AclEntryScope.ACCESS)
+ .setType(AclEntryType.OTHER)
+ .setPermission(perm.getOtherAction())
+ .build());
+ }
+
+ /**
+ * Checks if the given entries represent a minimal ACL (contains exactly 3
+ * entries).
+ *
+ * @param entries List<AclEntry> entries to check
+ * @return boolean true if the entries represent a minimal ACL
+ */
+ public static boolean isMinimalAcl(List<AclEntry> entries) {
+ return entries.size() == 3;
+ }
+
+ /**
+ * There is no reason to instantiate this class.
+ */
+ private AclUtil() {
+ }
+}
Added:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/ScopedAclEntries.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/ScopedAclEntries.java?rev=1595283&view=auto
==============================================================================
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/ScopedAclEntries.java
(added)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/ScopedAclEntries.java
Fri May 16 18:25:23 2014
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.permission;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.permission.AclEntry;
+import org.apache.hadoop.fs.permission.AclEntryScope;
+
+/**
+ * Groups a list of ACL entries into separate lists for access entries vs.
+ * default entries.
+ */
+@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
+@InterfaceStability.Unstable
+public final class ScopedAclEntries {
+ private static final int PIVOT_NOT_FOUND = -1;
+
+ private final List<AclEntry> accessEntries;
+ private final List<AclEntry> defaultEntries;
+
+ /**
+ * Creates a new ScopedAclEntries from the given list. It is assumed that
the
+ * list is already sorted such that all access entries precede all default
+ * entries.
+ *
+ * @param aclEntries List<AclEntry> to separate
+ */
+ public ScopedAclEntries(List<AclEntry> aclEntries) {
+ int pivot = calculatePivotOnDefaultEntries(aclEntries);
+ if (pivot != PIVOT_NOT_FOUND) {
+ accessEntries = pivot != 0 ? aclEntries.subList(0, pivot) :
+ Collections.<AclEntry>emptyList();
+ defaultEntries = aclEntries.subList(pivot, aclEntries.size());
+ } else {
+ accessEntries = aclEntries;
+ defaultEntries = Collections.emptyList();
+ }
+ }
+
+ /**
+ * Returns access entries.
+ *
+ * @return List<AclEntry> containing just access entries, or an empty list if
+ * there are no access entries
+ */
+ public List<AclEntry> getAccessEntries() {
+ return accessEntries;
+ }
+
+ /**
+ * Returns default entries.
+ *
+ * @return List<AclEntry> containing just default entries, or an empty list
if
+ * there are no default entries
+ */
+ public List<AclEntry> getDefaultEntries() {
+ return defaultEntries;
+ }
+
+ /**
+ * Returns the pivot point in the list between the access entries and the
+ * default entries. This is the index of the first element in the list that
is
+ * a default entry.
+ *
+ * @param aclBuilder ArrayList<AclEntry> containing entries to build
+ * @return int pivot point, or -1 if list contains no default entries
+ */
+ private static int calculatePivotOnDefaultEntries(List<AclEntry> aclBuilder)
{
+ for (int i = 0; i < aclBuilder.size(); ++i) {
+ if (aclBuilder.get(i).getScope() == AclEntryScope.DEFAULT) {
+ return i;
+ }
+ }
+ return PIVOT_NOT_FOUND;
+ }
+}
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java?rev=1595283&r1=1595282&r2=1595283&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java
Fri May 16 18:25:23 2014
@@ -18,7 +18,7 @@
package org.apache.hadoop.fs.shell;
import java.io.IOException;
-import java.util.Iterator;
+import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
@@ -31,8 +31,10 @@ import org.apache.hadoop.fs.permission.A
import org.apache.hadoop.fs.permission.AclEntryScope;
import org.apache.hadoop.fs.permission.AclEntryType;
import org.apache.hadoop.fs.permission.AclStatus;
+import org.apache.hadoop.fs.permission.AclUtil;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.permission.ScopedAclEntries;
/**
* Acl related operations
@@ -84,67 +86,34 @@ class AclCommands extends FsCommand {
(perm.getOtherAction().implies(FsAction.EXECUTE) ? "t" : "T"));
}
- if (perm.getAclBit()) {
- AclStatus aclStatus = item.fs.getAclStatus(item.path);
- List<AclEntry> entries = aclStatus.getEntries();
- printExtendedAcl(perm, entries);
- } else {
- printMinimalAcl(perm);
- }
-
+ List<AclEntry> entries = perm.getAclBit() ?
+ item.fs.getAclStatus(item.path).getEntries() :
+ Collections.<AclEntry>emptyList();
+ ScopedAclEntries scopedEntries = new ScopedAclEntries(
+ AclUtil.getAclFromPermAndEntries(perm, entries));
+ printAclEntriesForSingleScope(scopedEntries.getAccessEntries());
+ printAclEntriesForSingleScope(scopedEntries.getDefaultEntries());
out.println();
}
/**
- * Prints an extended ACL, including all extended ACL entries and also the
- * base entries implied by the permission bits.
+ * Prints all the ACL entries in a single scope.
*
- * @param perm FsPermission of file
* @param entries List<AclEntry> containing ACL entries of file
*/
- private void printExtendedAcl(FsPermission perm, List<AclEntry> entries) {
- // Print owner entry implied by owner permission bits.
- out.println(new AclEntry.Builder()
- .setScope(AclEntryScope.ACCESS)
- .setType(AclEntryType.USER)
- .setPermission(perm.getUserAction())
- .build());
-
- // Print all extended access ACL entries.
- boolean hasAccessAcl = false;
- Iterator<AclEntry> entryIter = entries.iterator();
- AclEntry curEntry = null;
- while (entryIter.hasNext()) {
- curEntry = entryIter.next();
- if (curEntry.getScope() == AclEntryScope.DEFAULT) {
- break;
- }
- hasAccessAcl = true;
- printExtendedAclEntry(curEntry, perm.getGroupAction());
+ private void printAclEntriesForSingleScope(List<AclEntry> entries) {
+ if (entries.isEmpty()) {
+ return;
}
-
- // Print mask entry implied by group permission bits, or print group
entry
- // if there is no access ACL (only default ACL).
- out.println(new AclEntry.Builder()
- .setScope(AclEntryScope.ACCESS)
- .setType(hasAccessAcl ? AclEntryType.MASK : AclEntryType.GROUP)
- .setPermission(perm.getGroupAction())
- .build());
-
- // Print other entry implied by other bits.
- out.println(new AclEntry.Builder()
- .setScope(AclEntryScope.ACCESS)
- .setType(AclEntryType.OTHER)
- .setPermission(perm.getOtherAction())
- .build());
-
- // Print default ACL entries.
- if (curEntry != null && curEntry.getScope() == AclEntryScope.DEFAULT) {
- out.println(curEntry);
- // ACL sort order guarantees default mask is the second-to-last entry.
+ if (AclUtil.isMinimalAcl(entries)) {
+ for (AclEntry entry: entries) {
+ out.println(entry);
+ }
+ } else {
+ // ACL sort order guarantees mask is the second-to-last entry.
FsAction maskPerm = entries.get(entries.size() - 2).getPermission();
- while (entryIter.hasNext()) {
- printExtendedAclEntry(entryIter.next(), maskPerm);
+ for (AclEntry entry: entries) {
+ printExtendedAclEntry(entry, maskPerm);
}
}
}
@@ -172,30 +141,6 @@ class AclCommands extends FsCommand {
out.println(entry);
}
}
-
- /**
- * Prints a minimal ACL, consisting of exactly 3 ACL entries implied by the
- * permission bits.
- *
- * @param perm FsPermission of file
- */
- private void printMinimalAcl(FsPermission perm) {
- out.println(new AclEntry.Builder()
- .setScope(AclEntryScope.ACCESS)
- .setType(AclEntryType.USER)
- .setPermission(perm.getUserAction())
- .build());
- out.println(new AclEntry.Builder()
- .setScope(AclEntryScope.ACCESS)
- .setType(AclEntryType.GROUP)
- .setPermission(perm.getGroupAction())
- .build());
- out.println(new AclEntry.Builder()
- .setScope(AclEntryScope.ACCESS)
- .setType(AclEntryType.OTHER)
- .setPermission(perm.getOtherAction())
- .build());
- }
}
/**
