Author: cnauroth Date: Fri May 16 18:25:23 2014 New Revision: 1595283 URL: http://svn.apache.org/r1595283 Log: MAPREDUCE-5809. Enhance distcp to support preserving HDFS ACLs. Contributed by Chris Nauroth.
Added: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclUtil.java hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/ScopedAclEntries.java Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java?rev=1595283&r1=1595282&r2=1595283&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java Fri May 16 18:25:23 2014 @@ -100,6 +100,21 @@ public class FileStatus implements Writa } /** + * Copy constructor. + * + * @param other FileStatus to copy + */ + public FileStatus(FileStatus other) throws IOException { + // It's important to call the getters here instead of directly accessing the + // members. Subclasses like ViewFsFileStatus can override the getters. + this(other.getLen(), other.isDirectory(), other.getReplication(), + other.getBlockSize(), other.getModificationTime(), other.getAccessTime(), + other.getPermission(), other.getOwner(), other.getGroup(), + (other.isSymlink() ? other.getSymlink() : null), + other.getPath()); + } + + /** * Get the length of this file, in bytes. * @return the length of this file, in bytes. */ Added: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclUtil.java?rev=1595283&view=auto ============================================================================== --- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclUtil.java (added) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclUtil.java Fri May 16 18:25:23 2014 @@ -0,0 +1,134 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.permission; + +import java.util.Iterator; +import java.util.List; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +import com.google.common.collect.Lists; + +/** + * AclUtil contains utility methods for manipulating ACLs. + */ +@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"}) +@InterfaceStability.Unstable +public final class AclUtil { + + /** + * Given permissions and extended ACL entries, returns the full logical ACL. + * + * @param perm FsPermission containing permissions + * @param entries List<AclEntry> containing extended ACL entries + * @return List<AclEntry> containing full logical ACL + */ + public static List<AclEntry> getAclFromPermAndEntries(FsPermission perm, + List<AclEntry> entries) { + List<AclEntry> acl = Lists.newArrayListWithCapacity(entries.size() + 3); + + // Owner entry implied by owner permission bits. + acl.add(new AclEntry.Builder() + .setScope(AclEntryScope.ACCESS) + .setType(AclEntryType.USER) + .setPermission(perm.getUserAction()) + .build()); + + // All extended access ACL entries. + boolean hasAccessAcl = false; + Iterator<AclEntry> entryIter = entries.iterator(); + AclEntry curEntry = null; + while (entryIter.hasNext()) { + curEntry = entryIter.next(); + if (curEntry.getScope() == AclEntryScope.DEFAULT) { + break; + } + hasAccessAcl = true; + acl.add(curEntry); + } + + // Mask entry implied by group permission bits, or group entry if there is + // no access ACL (only default ACL). + acl.add(new AclEntry.Builder() + .setScope(AclEntryScope.ACCESS) + .setType(hasAccessAcl ? AclEntryType.MASK : AclEntryType.GROUP) + .setPermission(perm.getGroupAction()) + .build()); + + // Other entry implied by other bits. + acl.add(new AclEntry.Builder() + .setScope(AclEntryScope.ACCESS) + .setType(AclEntryType.OTHER) + .setPermission(perm.getOtherAction()) + .build()); + + // Default ACL entries. + if (curEntry != null && curEntry.getScope() == AclEntryScope.DEFAULT) { + acl.add(curEntry); + while (entryIter.hasNext()) { + acl.add(entryIter.next()); + } + } + + return acl; + } + + /** + * Translates the given permission bits to the equivalent minimal ACL. + * + * @param perm FsPermission to translate + * @return List<AclEntry> containing exactly 3 entries representing the owner, + * group and other permissions + */ + public static List<AclEntry> getMinimalAcl(FsPermission perm) { + return Lists.newArrayList( + new AclEntry.Builder() + .setScope(AclEntryScope.ACCESS) + .setType(AclEntryType.USER) + .setPermission(perm.getUserAction()) + .build(), + new AclEntry.Builder() + .setScope(AclEntryScope.ACCESS) + .setType(AclEntryType.GROUP) + .setPermission(perm.getGroupAction()) + .build(), + new AclEntry.Builder() + .setScope(AclEntryScope.ACCESS) + .setType(AclEntryType.OTHER) + .setPermission(perm.getOtherAction()) + .build()); + } + + /** + * Checks if the given entries represent a minimal ACL (contains exactly 3 + * entries). + * + * @param entries List<AclEntry> entries to check + * @return boolean true if the entries represent a minimal ACL + */ + public static boolean isMinimalAcl(List<AclEntry> entries) { + return entries.size() == 3; + } + + /** + * There is no reason to instantiate this class. + */ + private AclUtil() { + } +} Added: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/ScopedAclEntries.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/ScopedAclEntries.java?rev=1595283&view=auto ============================================================================== --- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/ScopedAclEntries.java (added) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/ScopedAclEntries.java Fri May 16 18:25:23 2014 @@ -0,0 +1,95 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.permission; + +import java.util.Collections; +import java.util.List; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.permission.AclEntry; +import org.apache.hadoop.fs.permission.AclEntryScope; + +/** + * Groups a list of ACL entries into separate lists for access entries vs. + * default entries. + */ +@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"}) +@InterfaceStability.Unstable +public final class ScopedAclEntries { + private static final int PIVOT_NOT_FOUND = -1; + + private final List<AclEntry> accessEntries; + private final List<AclEntry> defaultEntries; + + /** + * Creates a new ScopedAclEntries from the given list. It is assumed that the + * list is already sorted such that all access entries precede all default + * entries. + * + * @param aclEntries List<AclEntry> to separate + */ + public ScopedAclEntries(List<AclEntry> aclEntries) { + int pivot = calculatePivotOnDefaultEntries(aclEntries); + if (pivot != PIVOT_NOT_FOUND) { + accessEntries = pivot != 0 ? aclEntries.subList(0, pivot) : + Collections.<AclEntry>emptyList(); + defaultEntries = aclEntries.subList(pivot, aclEntries.size()); + } else { + accessEntries = aclEntries; + defaultEntries = Collections.emptyList(); + } + } + + /** + * Returns access entries. + * + * @return List<AclEntry> containing just access entries, or an empty list if + * there are no access entries + */ + public List<AclEntry> getAccessEntries() { + return accessEntries; + } + + /** + * Returns default entries. + * + * @return List<AclEntry> containing just default entries, or an empty list if + * there are no default entries + */ + public List<AclEntry> getDefaultEntries() { + return defaultEntries; + } + + /** + * Returns the pivot point in the list between the access entries and the + * default entries. This is the index of the first element in the list that is + * a default entry. + * + * @param aclBuilder ArrayList<AclEntry> containing entries to build + * @return int pivot point, or -1 if list contains no default entries + */ + private static int calculatePivotOnDefaultEntries(List<AclEntry> aclBuilder) { + for (int i = 0; i < aclBuilder.size(); ++i) { + if (aclBuilder.get(i).getScope() == AclEntryScope.DEFAULT) { + return i; + } + } + return PIVOT_NOT_FOUND; + } +} Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java?rev=1595283&r1=1595282&r2=1595283&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java Fri May 16 18:25:23 2014 @@ -18,7 +18,7 @@ package org.apache.hadoop.fs.shell; import java.io.IOException; -import java.util.Iterator; +import java.util.Collections; import java.util.LinkedList; import java.util.List; @@ -31,8 +31,10 @@ import org.apache.hadoop.fs.permission.A import org.apache.hadoop.fs.permission.AclEntryScope; import org.apache.hadoop.fs.permission.AclEntryType; import org.apache.hadoop.fs.permission.AclStatus; +import org.apache.hadoop.fs.permission.AclUtil; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.fs.permission.ScopedAclEntries; /** * Acl related operations @@ -84,67 +86,34 @@ class AclCommands extends FsCommand { (perm.getOtherAction().implies(FsAction.EXECUTE) ? "t" : "T")); } - if (perm.getAclBit()) { - AclStatus aclStatus = item.fs.getAclStatus(item.path); - List<AclEntry> entries = aclStatus.getEntries(); - printExtendedAcl(perm, entries); - } else { - printMinimalAcl(perm); - } - + List<AclEntry> entries = perm.getAclBit() ? + item.fs.getAclStatus(item.path).getEntries() : + Collections.<AclEntry>emptyList(); + ScopedAclEntries scopedEntries = new ScopedAclEntries( + AclUtil.getAclFromPermAndEntries(perm, entries)); + printAclEntriesForSingleScope(scopedEntries.getAccessEntries()); + printAclEntriesForSingleScope(scopedEntries.getDefaultEntries()); out.println(); } /** - * Prints an extended ACL, including all extended ACL entries and also the - * base entries implied by the permission bits. + * Prints all the ACL entries in a single scope. * - * @param perm FsPermission of file * @param entries List<AclEntry> containing ACL entries of file */ - private void printExtendedAcl(FsPermission perm, List<AclEntry> entries) { - // Print owner entry implied by owner permission bits. - out.println(new AclEntry.Builder() - .setScope(AclEntryScope.ACCESS) - .setType(AclEntryType.USER) - .setPermission(perm.getUserAction()) - .build()); - - // Print all extended access ACL entries. - boolean hasAccessAcl = false; - Iterator<AclEntry> entryIter = entries.iterator(); - AclEntry curEntry = null; - while (entryIter.hasNext()) { - curEntry = entryIter.next(); - if (curEntry.getScope() == AclEntryScope.DEFAULT) { - break; - } - hasAccessAcl = true; - printExtendedAclEntry(curEntry, perm.getGroupAction()); + private void printAclEntriesForSingleScope(List<AclEntry> entries) { + if (entries.isEmpty()) { + return; } - - // Print mask entry implied by group permission bits, or print group entry - // if there is no access ACL (only default ACL). - out.println(new AclEntry.Builder() - .setScope(AclEntryScope.ACCESS) - .setType(hasAccessAcl ? AclEntryType.MASK : AclEntryType.GROUP) - .setPermission(perm.getGroupAction()) - .build()); - - // Print other entry implied by other bits. - out.println(new AclEntry.Builder() - .setScope(AclEntryScope.ACCESS) - .setType(AclEntryType.OTHER) - .setPermission(perm.getOtherAction()) - .build()); - - // Print default ACL entries. - if (curEntry != null && curEntry.getScope() == AclEntryScope.DEFAULT) { - out.println(curEntry); - // ACL sort order guarantees default mask is the second-to-last entry. + if (AclUtil.isMinimalAcl(entries)) { + for (AclEntry entry: entries) { + out.println(entry); + } + } else { + // ACL sort order guarantees mask is the second-to-last entry. FsAction maskPerm = entries.get(entries.size() - 2).getPermission(); - while (entryIter.hasNext()) { - printExtendedAclEntry(entryIter.next(), maskPerm); + for (AclEntry entry: entries) { + printExtendedAclEntry(entry, maskPerm); } } } @@ -172,30 +141,6 @@ class AclCommands extends FsCommand { out.println(entry); } } - - /** - * Prints a minimal ACL, consisting of exactly 3 ACL entries implied by the - * permission bits. - * - * @param perm FsPermission of file - */ - private void printMinimalAcl(FsPermission perm) { - out.println(new AclEntry.Builder() - .setScope(AclEntryScope.ACCESS) - .setType(AclEntryType.USER) - .setPermission(perm.getUserAction()) - .build()); - out.println(new AclEntry.Builder() - .setScope(AclEntryScope.ACCESS) - .setType(AclEntryType.GROUP) - .setPermission(perm.getGroupAction()) - .build()); - out.println(new AclEntry.Builder() - .setScope(AclEntryScope.ACCESS) - .setType(AclEntryType.OTHER) - .setPermission(perm.getOtherAction()) - .build()); - } } /**