Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java Tue Aug 19 23:49:39 2014 @@ -32,6 +32,7 @@ import org.apache.hadoop.fs.permission.A import org.apache.hadoop.fs.permission.AclEntryType; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.util.ReadOnlyList; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.StringUtils; @@ -136,6 +137,7 @@ class FSPermissionChecker { * @param subAccess If path is a directory, * it is the access required of the path and all the sub-directories. * If path is not a directory, there is no effect. + * @param ignoreEmptyDir Ignore permission checking for empty directory? * @param resolveLink whether to resolve the final path component if it is * a symlink * @throws AccessControlException @@ -144,9 +146,9 @@ class FSPermissionChecker { * Guarded by {@link FSNamesystem#readLock()} * Caller of this method must hold that lock. */ - void checkPermission(String path, INodeDirectory root, boolean doCheckOwner, + void checkPermission(String path, FSDirectory dir, boolean doCheckOwner, FsAction ancestorAccess, FsAction parentAccess, FsAction access, - FsAction subAccess, boolean resolveLink) + FsAction subAccess, boolean ignoreEmptyDir, boolean resolveLink) throws AccessControlException, UnresolvedLinkException { if (LOG.isDebugEnabled()) { LOG.debug("ACCESS CHECK: " + this @@ -155,11 +157,12 @@ class FSPermissionChecker { + ", parentAccess=" + parentAccess + ", access=" + access + ", subAccess=" + subAccess + + ", ignoreEmptyDir=" + ignoreEmptyDir + ", resolveLink=" + resolveLink); } // check if (parentAccess != null) && file exists, then check sb // If resolveLink, the check is performed on the link target. - final INodesInPath inodesInPath = root.getINodesInPath(path, resolveLink); + final INodesInPath inodesInPath = dir.getINodesInPath(path, resolveLink); final int snapshotId = inodesInPath.getPathSnapshotId(); final INode[] inodes = inodesInPath.getINodes(); int ancestorIndex = inodes.length - 2; @@ -182,7 +185,7 @@ class FSPermissionChecker { check(last, snapshotId, access); } if (subAccess != null) { - checkSubAccess(last, snapshotId, subAccess); + checkSubAccess(last, snapshotId, subAccess, ignoreEmptyDir); } if (doCheckOwner) { checkOwner(last, snapshotId); @@ -207,8 +210,8 @@ class FSPermissionChecker { } /** Guarded by {@link FSNamesystem#readLock()} */ - private void checkSubAccess(INode inode, int snapshotId, FsAction access - ) throws AccessControlException { + private void checkSubAccess(INode inode, int snapshotId, FsAction access, + boolean ignoreEmptyDir) throws AccessControlException { if (inode == null || !inode.isDirectory()) { return; } @@ -216,9 +219,12 @@ class FSPermissionChecker { Stack<INodeDirectory> directories = new Stack<INodeDirectory>(); for(directories.push(inode.asDirectory()); !directories.isEmpty(); ) { INodeDirectory d = directories.pop(); - check(d, snapshotId, access); + ReadOnlyList<INode> cList = d.getChildrenList(snapshotId); + if (!(cList.isEmpty() && ignoreEmptyDir)) { + check(d, snapshotId, access); + } - for(INode child : d.getChildrenList(snapshotId)) { + for(INode child : cList) { if (child.isDirectory()) { directories.push(child.asDirectory()); }
Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java Tue Aug 19 23:49:39 2014 @@ -43,6 +43,8 @@ import org.apache.hadoop.hdfs.server.nam import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeFile; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog; +import org.apache.hadoop.io.nativeio.NativeIO; + import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; @@ -69,6 +71,8 @@ public class FileJournalManager implemen NameNodeFile.EDITS.getName() + "_(\\d+)-(\\d+)"); private static final Pattern EDITS_INPROGRESS_REGEX = Pattern.compile( NameNodeFile.EDITS_INPROGRESS.getName() + "_(\\d+)"); + private static final Pattern EDITS_INPROGRESS_STALE_REGEX = Pattern.compile( + NameNodeFile.EDITS_INPROGRESS.getName() + "_(\\d+).*(\\S+)"); private File currentInProgress = null; @@ -132,10 +136,14 @@ public class FileJournalManager implemen Preconditions.checkState(!dstFile.exists(), "Can't finalize edits file " + inprogressFile + " since finalized file " + "already exists"); - if (!inprogressFile.renameTo(dstFile)) { + + try { + NativeIO.renameTo(inprogressFile, dstFile); + } catch (IOException e) { errorReporter.reportErrorOnFile(dstFile); - throw new IllegalStateException("Unable to finalize edits file " + inprogressFile); + throw new IllegalStateException("Unable to finalize edits file " + inprogressFile, e); } + if (inprogressFile.equals(currentInProgress)) { currentInProgress = null; } @@ -156,8 +164,7 @@ public class FileJournalManager implemen throws IOException { LOG.info("Purging logs older than " + minTxIdToKeep); File[] files = FileUtil.listFiles(sd.getCurrentDir()); - List<EditLogFile> editLogs = - FileJournalManager.matchEditLogs(files); + List<EditLogFile> editLogs = matchEditLogs(files, true); for (EditLogFile log : editLogs) { if (log.getFirstTxId() < minTxIdToKeep && log.getLastTxId() < minTxIdToKeep) { @@ -168,7 +175,7 @@ public class FileJournalManager implemen /** * Find all editlog segments starting at or above the given txid. - * @param fromTxId the txnid which to start looking + * @param firstTxId the txnid which to start looking * @param inProgressOk whether or not to include the in-progress edit log * segment * @return a list of remote edit logs @@ -238,8 +245,13 @@ public class FileJournalManager implemen public static List<EditLogFile> matchEditLogs(File logDir) throws IOException { return matchEditLogs(FileUtil.listFiles(logDir)); } - + static List<EditLogFile> matchEditLogs(File[] filesInStorage) { + return matchEditLogs(filesInStorage, false); + } + + private static List<EditLogFile> matchEditLogs(File[] filesInStorage, + boolean forPurging) { List<EditLogFile> ret = Lists.newArrayList(); for (File f : filesInStorage) { String name = f.getName(); @@ -250,6 +262,7 @@ public class FileJournalManager implemen long startTxId = Long.parseLong(editsMatch.group(1)); long endTxId = Long.parseLong(editsMatch.group(2)); ret.add(new EditLogFile(f, startTxId, endTxId)); + continue; } catch (NumberFormatException nfe) { LOG.error("Edits file " + f + " has improperly formatted " + "transaction ID"); @@ -264,12 +277,30 @@ public class FileJournalManager implemen long startTxId = Long.parseLong(inProgressEditsMatch.group(1)); ret.add( new EditLogFile(f, startTxId, HdfsConstants.INVALID_TXID, true)); + continue; } catch (NumberFormatException nfe) { LOG.error("In-progress edits file " + f + " has improperly " + "formatted transaction ID"); // skip } } + if (forPurging) { + // Check for in-progress stale edits + Matcher staleInprogressEditsMatch = EDITS_INPROGRESS_STALE_REGEX + .matcher(name); + if (staleInprogressEditsMatch.matches()) { + try { + long startTxId = Long.valueOf(staleInprogressEditsMatch.group(1)); + ret.add(new EditLogFile(f, startTxId, HdfsConstants.INVALID_TXID, + true)); + continue; + } catch (NumberFormatException nfe) { + LOG.error("In-progress stale edits file " + f + " has improperly " + + "formatted transaction ID"); + // skip + } + } + } } return ret; } @@ -513,11 +544,16 @@ public class FileJournalManager implemen File src = file; File dst = new File(src.getParent(), src.getName() + newSuffix); // renameTo fails on Windows if the destination file already exists. - if (!src.renameTo(dst)) { - if (!dst.delete() || !src.renameTo(dst)) { - throw new IOException( - "Couldn't rename log " + src + " to " + dst); + try { + if (dst.exists()) { + if (!dst.delete()) { + throw new IOException("Couldn't delete " + dst); + } } + NativeIO.renameTo(src, dst); + } catch (IOException e) { + throw new IOException( + "Couldn't rename log " + src + " to " + dst, e); } file = dst; } Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java Tue Aug 19 23:49:39 2014 @@ -32,15 +32,10 @@ import org.apache.hadoop.hdfs.server.nam public class FileUnderConstructionFeature implements INode.Feature { private String clientName; // lease holder private final String clientMachine; - // if client is a cluster node too. - private final DatanodeDescriptor clientNode; - public FileUnderConstructionFeature(final String clientName, - final String clientMachine, - final DatanodeDescriptor clientNode) { + public FileUnderConstructionFeature(final String clientName, final String clientMachine) { this.clientName = clientName; this.clientMachine = clientMachine; - this.clientNode = clientNode; } public String getClientName() { @@ -55,10 +50,6 @@ public class FileUnderConstructionFeatur return clientMachine; } - public DatanodeDescriptor getClientNode() { - return clientNode; - } - /** * Update the length for the last block * Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java Tue Aug 19 23:49:39 2014 @@ -20,7 +20,6 @@ package org.apache.hadoop.hdfs.server.na import java.io.PrintStream; import java.io.PrintWriter; import java.io.StringWriter; -import java.util.ArrayList; import java.util.List; import org.apache.commons.logging.Log; @@ -98,9 +97,9 @@ public abstract class INode implements I /** Set user */ final INode setUser(String user, int latestSnapshotId) throws QuotaExceededException { - final INode nodeToUpdate = recordModification(latestSnapshotId); - nodeToUpdate.setUser(user); - return nodeToUpdate; + recordModification(latestSnapshotId); + setUser(user); + return this; } /** * @param snapshotId @@ -123,9 +122,9 @@ public abstract class INode implements I /** Set group */ final INode setGroup(String group, int latestSnapshotId) throws QuotaExceededException { - final INode nodeToUpdate = recordModification(latestSnapshotId); - nodeToUpdate.setGroup(group); - return nodeToUpdate; + recordModification(latestSnapshotId); + setGroup(group); + return this; } /** @@ -149,9 +148,9 @@ public abstract class INode implements I /** Set the {@link FsPermission} of this {@link INode} */ INode setPermission(FsPermission permission, int latestSnapshotId) throws QuotaExceededException { - final INode nodeToUpdate = recordModification(latestSnapshotId); - nodeToUpdate.setPermission(permission); - return nodeToUpdate; + recordModification(latestSnapshotId); + setPermission(permission); + return this; } abstract AclFeature getAclFeature(int snapshotId); @@ -165,18 +164,56 @@ public abstract class INode implements I final INode addAclFeature(AclFeature aclFeature, int latestSnapshotId) throws QuotaExceededException { - final INode nodeToUpdate = recordModification(latestSnapshotId); - nodeToUpdate.addAclFeature(aclFeature); - return nodeToUpdate; + recordModification(latestSnapshotId); + addAclFeature(aclFeature); + return this; } abstract void removeAclFeature(); final INode removeAclFeature(int latestSnapshotId) throws QuotaExceededException { - final INode nodeToUpdate = recordModification(latestSnapshotId); - nodeToUpdate.removeAclFeature(); - return nodeToUpdate; + recordModification(latestSnapshotId); + removeAclFeature(); + return this; + } + + /** + * @param snapshotId + * if it is not {@link Snapshot#CURRENT_STATE_ID}, get the result + * from the given snapshot; otherwise, get the result from the + * current inode. + * @return XAttrFeature + */ + abstract XAttrFeature getXAttrFeature(int snapshotId); + + @Override + public final XAttrFeature getXAttrFeature() { + return getXAttrFeature(Snapshot.CURRENT_STATE_ID); + } + + /** + * Set <code>XAttrFeature</code> + */ + abstract void addXAttrFeature(XAttrFeature xAttrFeature); + + final INode addXAttrFeature(XAttrFeature xAttrFeature, int latestSnapshotId) + throws QuotaExceededException { + recordModification(latestSnapshotId); + addXAttrFeature(xAttrFeature); + return this; + } + + /** + * Remove <code>XAttrFeature</code> + */ + abstract void removeXAttrFeature(); + + final INode removeXAttrFeature(int lastestSnapshotId) + throws QuotaExceededException { + recordModification(lastestSnapshotId); + removeXAttrFeature(); + return this; } /** @@ -261,11 +298,8 @@ public abstract class INode implements I * @param latestSnapshotId The id of the latest snapshot that has been taken. * Note that it is {@link Snapshot#CURRENT_STATE_ID} * if no snapshots have been taken. - * @return The current inode, which usually is the same object of this inode. - * However, in some cases, this inode may be replaced with a new inode - * for maintaining snapshots. The current inode is then the new inode. */ - abstract INode recordModification(final int latestSnapshotId) + abstract void recordModification(final int latestSnapshotId) throws QuotaExceededException; /** Check whether it's a reference. */ @@ -615,9 +649,9 @@ public abstract class INode implements I /** Set the last modification time of inode. */ public final INode setModificationTime(long modificationTime, int latestSnapshotId) throws QuotaExceededException { - final INode nodeToUpdate = recordModification(latestSnapshotId); - nodeToUpdate.setModificationTime(modificationTime); - return nodeToUpdate; + recordModification(latestSnapshotId); + setModificationTime(modificationTime); + return this; } /** @@ -645,19 +679,19 @@ public abstract class INode implements I */ public final INode setAccessTime(long accessTime, int latestSnapshotId) throws QuotaExceededException { - final INode nodeToUpdate = recordModification(latestSnapshotId); - nodeToUpdate.setAccessTime(accessTime); - return nodeToUpdate; + recordModification(latestSnapshotId); + setAccessTime(accessTime); + return this; } /** - * Breaks file path into components. - * @param path - * @return array of byte arrays each of which represents + * Breaks {@code path} into components. + * @return array of byte arrays each of which represents * a single path component. */ - static byte[][] getPathComponents(String path) { + @VisibleForTesting + public static byte[][] getPathComponents(String path) { return getPathComponents(getPathNames(path)); } @@ -673,8 +707,7 @@ public abstract class INode implements I } /** - * Splits an absolute path into an array of path components. - * @param path + * Splits an absolute {@code path} into an array of path components. * @throws AssertionError if the given path is invalid. * @return array of path components. */ Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeAttributes.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeAttributes.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeAttributes.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeAttributes.java Tue Aug 19 23:49:39 2014 @@ -21,6 +21,7 @@ import org.apache.hadoop.classification. import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.permission.PermissionStatus; import org.apache.hadoop.hdfs.server.namenode.INodeWithAdditionalFields.PermissionStatusFormat; +import org.apache.hadoop.hdfs.server.namenode.XAttrFeature; /** * The attributes of an inode. @@ -50,6 +51,9 @@ public interface INodeAttributes { /** @return the ACL feature. */ public AclFeature getAclFeature(); + + /** @return the XAttrs feature. */ + public XAttrFeature getXAttrFeature(); /** @return the modification time. */ public long getModificationTime(); @@ -64,14 +68,17 @@ public interface INodeAttributes { private final AclFeature aclFeature; private final long modificationTime; private final long accessTime; + private XAttrFeature xAttrFeature; SnapshotCopy(byte[] name, PermissionStatus permissions, - AclFeature aclFeature, long modificationTime, long accessTime) { + AclFeature aclFeature, long modificationTime, long accessTime, + XAttrFeature xAttrFeature) { this.name = name; this.permission = PermissionStatusFormat.toLong(permissions); this.aclFeature = aclFeature; this.modificationTime = modificationTime; this.accessTime = accessTime; + this.xAttrFeature = xAttrFeature; } SnapshotCopy(INode inode) { @@ -80,6 +87,7 @@ public interface INodeAttributes { this.aclFeature = inode.getAclFeature(); this.modificationTime = inode.getModificationTime(); this.accessTime = inode.getAccessTime(); + this.xAttrFeature = inode.getXAttrFeature(); } @Override @@ -89,14 +97,12 @@ public interface INodeAttributes { @Override public final String getUserName() { - final int n = (int)PermissionStatusFormat.USER.retrieve(permission); - return SerialNumberManager.INSTANCE.getUser(n); + return PermissionStatusFormat.getUser(permission); } @Override public final String getGroupName() { - final int n = (int)PermissionStatusFormat.GROUP.retrieve(permission); - return SerialNumberManager.INSTANCE.getGroup(n); + return PermissionStatusFormat.getGroup(permission); } @Override @@ -106,7 +112,7 @@ public interface INodeAttributes { @Override public final short getFsPermissionShort() { - return (short)PermissionStatusFormat.MODE.retrieve(permission); + return PermissionStatusFormat.getMode(permission); } @Override @@ -128,5 +134,10 @@ public interface INodeAttributes { public final long getAccessTime() { return accessTime; } + + @Override + public final XAttrFeature getXAttrFeature() { + return xAttrFeature; + } } } Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java Tue Aug 19 23:49:39 2014 @@ -26,15 +26,14 @@ import java.util.List; import java.util.Map; import org.apache.hadoop.fs.PathIsNotDirectoryException; -import org.apache.hadoop.fs.UnresolvedLinkException; import org.apache.hadoop.fs.permission.PermissionStatus; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.protocol.QuotaExceededException; -import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException; +import org.apache.hadoop.hdfs.protocol.SnapshotException; import org.apache.hadoop.hdfs.server.namenode.INodeReference.WithCount; +import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectorySnapshottableFeature; import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature; import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiffList; -import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; import org.apache.hadoop.hdfs.util.Diff.ListType; import org.apache.hadoop.hdfs.util.ReadOnlyList; @@ -104,11 +103,6 @@ public class INodeDirectory extends INod return this; } - /** Is this a snapshottable directory? */ - public boolean isSnapshottable() { - return false; - } - void setQuota(long nsQuota, long dsQuota) { DirectoryWithQuotaFeature quota = getDirectoryWithQuotaFeature(); if (quota != null) { @@ -163,7 +157,7 @@ public class INodeDirectory extends INod return quota; } - private int searchChildren(byte[] name) { + int searchChildren(byte[] name) { return children == null? -1: Collections.binarySearch(children, name); } @@ -188,7 +182,7 @@ public class INodeDirectory extends INod public final boolean isWithSnapshot() { return getDirectoryWithSnapshotFeature() != null; } - + public DirectoryDiffList getDiffs() { DirectoryWithSnapshotFeature sf = getDirectoryWithSnapshotFeature(); return sf != null ? sf.getDiffs() : null; @@ -206,50 +200,71 @@ public class INodeDirectory extends INod return super.toDetailString() + (sf == null ? "" : ", " + sf.getDiffs()); } - /** Replace itself with an {@link INodeDirectorySnapshottable}. */ - public INodeDirectorySnapshottable replaceSelf4INodeDirectorySnapshottable( - int latestSnapshotId, final INodeMap inodeMap) - throws QuotaExceededException { - Preconditions.checkState(!(this instanceof INodeDirectorySnapshottable), - "this is already an INodeDirectorySnapshottable, this=%s", this); - final INodeDirectorySnapshottable s = new INodeDirectorySnapshottable(this); - replaceSelf(s, inodeMap).getDirectoryWithSnapshotFeature().getDiffs() - .saveSelf2Snapshot(latestSnapshotId, s, this); - return s; - } - - /** Replace itself with {@link INodeDirectory}. */ - public INodeDirectory replaceSelf4INodeDirectory(final INodeMap inodeMap) { - Preconditions.checkState(getClass() != INodeDirectory.class, - "the class is already INodeDirectory, this=%s", this); - return replaceSelf(new INodeDirectory(this, true, this.getFeatures()), - inodeMap); + public DirectorySnapshottableFeature getDirectorySnapshottableFeature() { + return getFeature(DirectorySnapshottableFeature.class); } - /** Replace itself with the given directory. */ - private final <N extends INodeDirectory> N replaceSelf(final N newDir, - final INodeMap inodeMap) { - final INodeReference ref = getParentReference(); - if (ref != null) { - ref.setReferredINode(newDir); - if (inodeMap != null) { - inodeMap.put(newDir); - } - } else { - final INodeDirectory parent = getParent(); - Preconditions.checkArgument(parent != null, "parent is null, this=%s", this); - parent.replaceChild(this, newDir, inodeMap); + public boolean isSnapshottable() { + return getDirectorySnapshottableFeature() != null; + } + + public Snapshot getSnapshot(byte[] snapshotName) { + return getDirectorySnapshottableFeature().getSnapshot(snapshotName); + } + + public void setSnapshotQuota(int snapshotQuota) { + getDirectorySnapshottableFeature().setSnapshotQuota(snapshotQuota); + } + + public Snapshot addSnapshot(int id, String name) throws SnapshotException, + QuotaExceededException { + return getDirectorySnapshottableFeature().addSnapshot(this, id, name); + } + + public Snapshot removeSnapshot(String snapshotName, + BlocksMapUpdateInfo collectedBlocks, final List<INode> removedINodes) + throws SnapshotException { + return getDirectorySnapshottableFeature().removeSnapshot(this, + snapshotName, collectedBlocks, removedINodes); + } + + public void renameSnapshot(String path, String oldName, String newName) + throws SnapshotException { + getDirectorySnapshottableFeature().renameSnapshot(path, oldName, newName); + } + + /** add DirectorySnapshottableFeature */ + public void addSnapshottableFeature() { + Preconditions.checkState(!isSnapshottable(), + "this is already snapshottable, this=%s", this); + DirectoryWithSnapshotFeature s = this.getDirectoryWithSnapshotFeature(); + final DirectorySnapshottableFeature snapshottable = + new DirectorySnapshottableFeature(s); + if (s != null) { + this.removeFeature(s); } - clear(); - return newDir; + this.addFeature(snapshottable); } - + + /** remove DirectorySnapshottableFeature */ + public void removeSnapshottableFeature() { + DirectorySnapshottableFeature s = getDirectorySnapshottableFeature(); + Preconditions.checkState(s != null, + "The dir does not have snapshottable feature: this=%s", this); + this.removeFeature(s); + if (s.getDiffs().asList().size() > 0) { + // add a DirectoryWithSnapshotFeature back + DirectoryWithSnapshotFeature sf = new DirectoryWithSnapshotFeature( + s.getDiffs()); + addFeature(sf); + } + } + /** * Replace the given child with a new child. Note that we no longer need to * replace an normal INodeDirectory or INodeFile into an * INodeDirectoryWithSnapshot or INodeFileUnderConstruction. The only cases - * for child replacement is for {@link INodeDirectorySnapshottable} and - * reference nodes. + * for child replacement is for reference nodes. */ public void replaceChild(INode oldChild, final INode newChild, final INodeMap inodeMap) { @@ -303,7 +318,7 @@ public class INodeDirectory extends INod } @Override - public INodeDirectory recordModification(int latestSnapshotId) + public void recordModification(int latestSnapshotId) throws QuotaExceededException { if (isInLatestSnapshot(latestSnapshotId) && !shouldRecordInSrcSnapshot(latestSnapshotId)) { @@ -315,7 +330,6 @@ public class INodeDirectory extends INod // record self in the diff list if necessary sf.getDiffs().saveSelf2Snapshot(latestSnapshotId, this, null); } - return this; } /** @@ -356,6 +370,29 @@ public class INodeDirectory extends INod return sf.getChild(this, name, snapshotId); } + + /** + * Search for the given INode in the children list and the deleted lists of + * snapshots. + * @return {@link Snapshot#CURRENT_STATE_ID} if the inode is in the children + * list; {@link Snapshot#NO_SNAPSHOT_ID} if the inode is neither in the + * children list nor in any snapshot; otherwise the snapshot id of the + * corresponding snapshot diff list. + */ + public int searchChild(INode inode) { + INode child = getChild(inode.getLocalNameBytes(), Snapshot.CURRENT_STATE_ID); + if (child != inode) { + // inode is not in parent's children list, thus inode must be in + // snapshot. identify the snapshot id and later add it into the path + DirectoryDiffList diffs = getDiffs(); + if (diffs == null) { + return Snapshot.NO_SNAPSHOT_ID; + } + return diffs.findSnapshotDeleted(inode); + } else { + return Snapshot.CURRENT_STATE_ID; + } + } /** * @param snapshotId @@ -380,53 +417,6 @@ public class INodeDirectory extends INod : ReadOnlyList.Util.asReadOnlyList(children); } - /** @return the {@link INodesInPath} containing only the last inode. */ - INodesInPath getLastINodeInPath(String path, boolean resolveLink - ) throws UnresolvedLinkException { - return INodesInPath.resolve(this, getPathComponents(path), 1, resolveLink); - } - - /** @return the {@link INodesInPath} containing all inodes in the path. */ - INodesInPath getINodesInPath(String path, boolean resolveLink - ) throws UnresolvedLinkException { - final byte[][] components = getPathComponents(path); - return INodesInPath.resolve(this, components, components.length, resolveLink); - } - - /** @return the last inode in the path. */ - INode getNode(String path, boolean resolveLink) - throws UnresolvedLinkException { - return getLastINodeInPath(path, resolveLink).getINode(0); - } - - /** - * @return the INode of the last component in src, or null if the last - * component does not exist. - * @throws UnresolvedLinkException if symlink can't be resolved - * @throws SnapshotAccessControlException if path is in RO snapshot - */ - INode getINode4Write(String src, boolean resolveLink) - throws UnresolvedLinkException, SnapshotAccessControlException { - return getINodesInPath4Write(src, resolveLink).getLastINode(); - } - - /** - * @return the INodesInPath of the components in src - * @throws UnresolvedLinkException if symlink can't be resolved - * @throws SnapshotAccessControlException if path is in RO snapshot - */ - INodesInPath getINodesInPath4Write(String src, boolean resolveLink) - throws UnresolvedLinkException, SnapshotAccessControlException { - final byte[][] components = INode.getPathComponents(src); - INodesInPath inodesInPath = INodesInPath.resolve(this, components, - components.length, resolveLink); - if (inodesInPath.isSnapshot()) { - throw new SnapshotAccessControlException( - "Modification on a read-only snapshot is disallowed"); - } - return inodesInPath; - } - /** * Given a child's name, return the index of the next child * @@ -788,7 +778,9 @@ public class INodeDirectory extends INod public boolean metadataEquals(INodeDirectoryAttributes other) { return other != null && getQuotaCounts().equals(other.getQuotaCounts()) - && getPermissionLong() == other.getPermissionLong(); + && getPermissionLong() == other.getPermissionLong() + && getAclFeature() == other.getAclFeature() + && getXAttrFeature() == other.getXAttrFeature(); } /* @@ -846,6 +838,11 @@ public class INodeDirectory extends INod }; } }); + + final DirectorySnapshottableFeature s = getDirectorySnapshottableFeature(); + if (s != null) { + s.dumpTreeRecursively(this, out, prefix, snapshot); + } } /** @@ -854,7 +851,7 @@ public class INodeDirectory extends INod * @param subs The subtrees. */ @VisibleForTesting - protected static void dumpTreeRecursively(PrintWriter out, + public static void dumpTreeRecursively(PrintWriter out, StringBuilder prefix, Iterable<SnapshotAndINode> subs) { if (subs != null) { for(final Iterator<SnapshotAndINode> i = subs.iterator(); i.hasNext();) { @@ -867,7 +864,7 @@ public class INodeDirectory extends INod } /** A pair of Snapshot and INode objects. */ - protected static class SnapshotAndINode { + public static class SnapshotAndINode { public final int snapshotId; public final INode inode; Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryAttributes.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryAttributes.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryAttributes.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryAttributes.java Tue Aug 19 23:49:39 2014 @@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.na import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.permission.PermissionStatus; +import org.apache.hadoop.hdfs.server.namenode.XAttrFeature; import com.google.common.base.Preconditions; @@ -35,8 +36,9 @@ public interface INodeDirectoryAttribute public static class SnapshotCopy extends INodeAttributes.SnapshotCopy implements INodeDirectoryAttributes { public SnapshotCopy(byte[] name, PermissionStatus permissions, - AclFeature aclFeature, long modificationTime) { - super(name, permissions, aclFeature, modificationTime, 0L); + AclFeature aclFeature, long modificationTime, + XAttrFeature xAttrsFeature) { + super(name, permissions, aclFeature, modificationTime, 0L, xAttrsFeature); } public SnapshotCopy(INodeDirectory dir) { @@ -51,8 +53,10 @@ public interface INodeDirectoryAttribute @Override public boolean metadataEquals(INodeDirectoryAttributes other) { return other != null - && this.getQuotaCounts().equals(other.getQuotaCounts()) - && getPermissionLong() == other.getPermissionLong(); + && getQuotaCounts().equals(other.getQuotaCounts()) + && getPermissionLong() == other.getPermissionLong() + && getAclFeature() == other.getAclFeature() + && getXAttrFeature() == other.getXAttrFeature(); } } @@ -63,8 +67,8 @@ public interface INodeDirectoryAttribute public CopyWithQuota(byte[] name, PermissionStatus permissions, AclFeature aclFeature, long modificationTime, long nsQuota, - long dsQuota) { - super(name, permissions, aclFeature, modificationTime); + long dsQuota, XAttrFeature xAttrsFeature) { + super(name, permissions, aclFeature, modificationTime, xAttrsFeature); this.nsQuota = nsQuota; this.dsQuota = dsQuota; } Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java Tue Aug 19 23:49:39 2014 @@ -33,13 +33,13 @@ import org.apache.hadoop.hdfs.protocol.Q import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction; -import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState; import org.apache.hadoop.hdfs.server.namenode.snapshot.FileDiff; import org.apache.hadoop.hdfs.server.namenode.snapshot.FileDiffList; import org.apache.hadoop.hdfs.server.namenode.snapshot.FileWithSnapshotFeature; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; +import org.apache.hadoop.hdfs.util.LongBitFormat; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -72,37 +72,29 @@ public class INodeFile extends INodeWith } /** Format: [16 bits for replication][48 bits for PreferredBlockSize] */ - static class HeaderFormat { - /** Number of bits for Block size */ - static final int BLOCKBITS = 48; - /** Header mask 64-bit representation */ - static final long HEADERMASK = 0xffffL << BLOCKBITS; - static final long MAX_BLOCK_SIZE = ~HEADERMASK; - - static short getReplication(long header) { - return (short) ((header & HEADERMASK) >> BLOCKBITS); + static enum HeaderFormat { + PREFERRED_BLOCK_SIZE(null, 48, 1), + REPLICATION(PREFERRED_BLOCK_SIZE.BITS, 16, 1); + + private final LongBitFormat BITS; + + private HeaderFormat(LongBitFormat previous, int length, long min) { + BITS = new LongBitFormat(name(), previous, length, min); } - static long combineReplication(long header, short replication) { - if (replication <= 0) { - throw new IllegalArgumentException( - "Unexpected value for the replication: " + replication); - } - return ((long)replication << BLOCKBITS) | (header & MAX_BLOCK_SIZE); + static short getReplication(long header) { + return (short)REPLICATION.BITS.retrieve(header); } - + static long getPreferredBlockSize(long header) { - return header & MAX_BLOCK_SIZE; + return PREFERRED_BLOCK_SIZE.BITS.retrieve(header); } - static long combinePreferredBlockSize(long header, long blockSize) { - if (blockSize < 0) { - throw new IllegalArgumentException("Block size < 0: " + blockSize); - } else if (blockSize > MAX_BLOCK_SIZE) { - throw new IllegalArgumentException("Block size = " + blockSize - + " > MAX_BLOCK_SIZE = " + MAX_BLOCK_SIZE); - } - return (header & HEADERMASK) | (blockSize & MAX_BLOCK_SIZE); + static long toLong(long preferredBlockSize, short replication) { + long h = 0; + h = PREFERRED_BLOCK_SIZE.BITS.combine(preferredBlockSize, h); + h = REPLICATION.BITS.combine(replication, h); + return h; } } @@ -114,8 +106,7 @@ public class INodeFile extends INodeWith long atime, BlockInfo[] blklist, short replication, long preferredBlockSize) { super(id, name, permissions, mtime, atime); - header = HeaderFormat.combineReplication(header, replication); - header = HeaderFormat.combinePreferredBlockSize(header, preferredBlockSize); + header = HeaderFormat.toLong(preferredBlockSize, replication); this.blocks = blklist; } @@ -144,6 +135,15 @@ public class INodeFile extends INodeWith return this; } + @Override + public boolean metadataEquals(INodeFileAttributes other) { + return other != null + && getHeaderLong()== other.getHeaderLong() + && getPermissionLong() == other.getPermissionLong() + && getAclFeature() == other.getAclFeature() + && getXAttrFeature() == other.getXAttrFeature(); + } + /* Start of Under-Construction Feature */ /** @@ -161,12 +161,11 @@ public class INodeFile extends INodeWith } /** Convert this file to an {@link INodeFileUnderConstruction}. */ - INodeFile toUnderConstruction(String clientName, String clientMachine, - DatanodeDescriptor clientNode) { + INodeFile toUnderConstruction(String clientName, String clientMachine) { Preconditions.checkState(!isUnderConstruction(), "file is already under construction"); FileUnderConstructionFeature uc = new FileUnderConstructionFeature( - clientName, clientMachine, clientNode); + clientName, clientMachine); addFeature(uc); return this; } @@ -285,7 +284,7 @@ public class INodeFile extends INodeWith } @Override - public INodeFile recordModification(final int latestSnapshotId) + public void recordModification(final int latestSnapshotId) throws QuotaExceededException { if (isInLatestSnapshot(latestSnapshotId) && !shouldRecordInSrcSnapshot(latestSnapshotId)) { @@ -297,7 +296,6 @@ public class INodeFile extends INodeWith // record self in the diff list if necessary sf.getDiffs().saveSelf2Snapshot(latestSnapshotId, this, null); } - return this; } public FileDiffList getDiffs() { @@ -340,16 +338,15 @@ public class INodeFile extends INodeWith /** Set the replication factor of this file. */ public final void setFileReplication(short replication) { - header = HeaderFormat.combineReplication(header, replication); + header = HeaderFormat.REPLICATION.BITS.combine(replication, header); } /** Set the replication factor of this file. */ public final INodeFile setFileReplication(short replication, - int latestSnapshotId, final INodeMap inodeMap) - throws QuotaExceededException { - final INodeFile nodeToUpdate = recordModification(latestSnapshotId); - nodeToUpdate.setFileReplication(replication); - return nodeToUpdate; + int latestSnapshotId) throws QuotaExceededException { + recordModification(latestSnapshotId); + setFileReplication(replication); + return this; } /** @return preferred block size (in bytes) of the file. */ @@ -435,17 +432,19 @@ public class INodeFile extends INodeWith removedINodes, countDiffChange); } Quota.Counts counts = Quota.Counts.newInstance(); - if (snapshot == CURRENT_STATE_ID && priorSnapshotId == NO_SNAPSHOT_ID) { - // this only happens when deleting the current file and the file is not - // in any snapshot - computeQuotaUsage(counts, false); - destroyAndCollectBlocks(collectedBlocks, removedINodes); - } else if (snapshot == CURRENT_STATE_ID && priorSnapshotId != NO_SNAPSHOT_ID) { - // when deleting the current file and the file is in snapshot, we should - // clean the 0-sized block if the file is UC - FileUnderConstructionFeature uc = getFileUnderConstructionFeature(); - if (uc != null) { - uc.cleanZeroSizeBlock(this, collectedBlocks); + if (snapshot == CURRENT_STATE_ID) { + if (priorSnapshotId == NO_SNAPSHOT_ID) { + // this only happens when deleting the current file and the file is not + // in any snapshot + computeQuotaUsage(counts, false); + destroyAndCollectBlocks(collectedBlocks, removedINodes); + } else { + // when deleting the current file and the file is in snapshot, we should + // clean the 0-sized block if the file is UC + FileUnderConstructionFeature uc = getFileUnderConstructionFeature(); + if (uc != null) { + uc.cleanZeroSizeBlock(this, collectedBlocks); + } } } return counts; Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java Tue Aug 19 23:49:39 2014 @@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.na import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.permission.PermissionStatus; import org.apache.hadoop.hdfs.server.namenode.INodeFile.HeaderFormat; +import org.apache.hadoop.hdfs.server.namenode.XAttrFeature; /** * The attributes of a file. @@ -35,6 +36,8 @@ public interface INodeFileAttributes ext /** @return the header as a long. */ public long getHeaderLong(); + public boolean metadataEquals(INodeFileAttributes other); + /** A copy of the inode file attributes */ public static class SnapshotCopy extends INodeAttributes.SnapshotCopy implements INodeFileAttributes { @@ -42,11 +45,10 @@ public interface INodeFileAttributes ext public SnapshotCopy(byte[] name, PermissionStatus permissions, AclFeature aclFeature, long modificationTime, long accessTime, - short replication, long preferredBlockSize) { - super(name, permissions, aclFeature, modificationTime, accessTime); - - final long h = HeaderFormat.combineReplication(0L, replication); - header = HeaderFormat.combinePreferredBlockSize(h, preferredBlockSize); + short replication, long preferredBlockSize, XAttrFeature xAttrsFeature) { + super(name, permissions, aclFeature, modificationTime, accessTime, + xAttrsFeature); + header = HeaderFormat.toLong(preferredBlockSize, replication); } public SnapshotCopy(INodeFile file) { @@ -68,5 +70,14 @@ public interface INodeFileAttributes ext public long getHeaderLong() { return header; } + + @Override + public boolean metadataEquals(INodeFileAttributes other) { + return other != null + && getHeaderLong()== other.getHeaderLong() + && getPermissionLong() == other.getPermissionLong() + && getAclFeature() == other.getAclFeature() + && getXAttrFeature() == other.getXAttrFeature(); + } } } Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java Tue Aug 19 23:49:39 2014 @@ -93,9 +93,8 @@ public class INodeMap { "", "", new FsPermission((short) 0)), 0, 0) { @Override - INode recordModification(int latestSnapshotId) + void recordModification(int latestSnapshotId) throws QuotaExceededException { - return null; } @Override Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java Tue Aug 19 23:49:39 2014 @@ -28,6 +28,7 @@ import org.apache.hadoop.fs.permission.P import org.apache.hadoop.hdfs.protocol.QuotaExceededException; import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; +import org.apache.hadoop.hdfs.server.namenode.XAttrFeature; import com.google.common.base.Preconditions; @@ -39,7 +40,7 @@ import com.google.common.base.Preconditi * snapshots and it is renamed/moved to other locations. * * For example, - * (1) Support we have /abc/foo, say the inode of foo is inode(id=1000,name=foo) + * (1) Suppose we have /abc/foo, say the inode of foo is inode(id=1000,name=foo) * (2) create snapshot s0 for /abc * (3) mv /abc/foo /xyz/bar, i.e. inode(id=1000,name=...) is renamed from "foo" * to "bar" and its parent becomes /xyz. @@ -228,6 +229,21 @@ public abstract class INodeReference ext final void removeAclFeature() { referred.removeAclFeature(); } + + @Override + final XAttrFeature getXAttrFeature(int snapshotId) { + return referred.getXAttrFeature(snapshotId); + } + + @Override + final void addXAttrFeature(XAttrFeature xAttrFeature) { + referred.addXAttrFeature(xAttrFeature); + } + + @Override + final void removeXAttrFeature() { + referred.removeXAttrFeature(); + } @Override public final short getFsPermissionShort() { @@ -271,11 +287,9 @@ public abstract class INodeReference ext } @Override - final INode recordModification(int latestSnapshotId) + final void recordModification(int latestSnapshotId) throws QuotaExceededException { referred.recordModification(latestSnapshotId); - // reference is never replaced - return this; } @Override // used by WithCount @@ -418,6 +432,30 @@ public abstract class INodeReference ext return withNameList.get(-i - 2); } } + + /** + * @return the WithName/DstReference node contained in the given snapshot. + */ + public INodeReference getParentRef(int snapshotId) { + int start = 0; + int end = withNameList.size() - 1; + while (start < end) { + int mid = start + (end - start) / 2; + int sid = withNameList.get(mid).lastSnapshotId; + if (sid == snapshotId) { + return withNameList.get(mid); + } else if (sid < snapshotId) { + start = mid + 1; + } else { + end = mid; + } + } + if (withNameList.get(start).lastSnapshotId >= snapshotId) { + return withNameList.get(start); + } else { + return this.getParentReference(); + } + } } /** A reference with a fixed name. */ Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java Tue Aug 19 23:49:39 2014 @@ -25,6 +25,8 @@ import org.apache.hadoop.fs.permission.P import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.protocol.QuotaExceededException; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; +import org.apache.hadoop.hdfs.server.namenode.AclFeature; +import org.apache.hadoop.hdfs.server.namenode.XAttrFeature; /** * An {@link INode} representing a symbolic link. @@ -45,12 +47,11 @@ public class INodeSymlink extends INodeW } @Override - INode recordModification(int latestSnapshotId) throws QuotaExceededException { + void recordModification(int latestSnapshotId) throws QuotaExceededException { if (isInLatestSnapshot(latestSnapshotId)) { INodeDirectory parent = getParent(); parent.saveChild2Snapshot(this, latestSnapshotId, new INodeSymlink(this)); } - return this; } /** @return true unconditionally. */ @@ -110,4 +111,38 @@ public class INodeSymlink extends INodeW super.dumpTreeRecursively(out, prefix, snapshot); out.println(); } + + /** + * getAclFeature is not overridden because it is needed for resolving + * symlinks. + @Override + final AclFeature getAclFeature(int snapshotId) { + throw new UnsupportedOperationException("ACLs are not supported on symlinks"); + } + */ + + @Override + public void removeAclFeature() { + throw new UnsupportedOperationException("ACLs are not supported on symlinks"); + } + + @Override + public void addAclFeature(AclFeature f) { + throw new UnsupportedOperationException("ACLs are not supported on symlinks"); + } + + @Override + final XAttrFeature getXAttrFeature(int snapshotId) { + throw new UnsupportedOperationException("XAttrs are not supported on symlinks"); + } + + @Override + public void removeXAttrFeature() { + throw new UnsupportedOperationException("XAttrs are not supported on symlinks"); + } + + @Override + public void addXAttrFeature(XAttrFeature f) { + throw new UnsupportedOperationException("XAttrs are not supported on symlinks"); + } } Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java Tue Aug 19 23:49:39 2014 @@ -21,8 +21,8 @@ import org.apache.hadoop.classification. import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.permission.PermissionStatus; import org.apache.hadoop.hdfs.protocol.QuotaExceededException; -import org.apache.hadoop.hdfs.server.namenode.INode.Feature; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; +import org.apache.hadoop.hdfs.util.LongBitFormat; import org.apache.hadoop.util.LightWeightGSet.LinkedElement; import com.google.common.base.Preconditions; @@ -35,26 +35,28 @@ import com.google.common.base.Preconditi public abstract class INodeWithAdditionalFields extends INode implements LinkedElement { static enum PermissionStatusFormat { - MODE(0, 16), - GROUP(MODE.OFFSET + MODE.LENGTH, 25), - USER(GROUP.OFFSET + GROUP.LENGTH, 23); + MODE(null, 16), + GROUP(MODE.BITS, 25), + USER(GROUP.BITS, 23); - final int OFFSET; - final int LENGTH; //bit length - final long MASK; + final LongBitFormat BITS; - PermissionStatusFormat(int offset, int length) { - OFFSET = offset; - LENGTH = length; - MASK = ((-1L) >>> (64 - LENGTH)) << OFFSET; + private PermissionStatusFormat(LongBitFormat previous, int length) { + BITS = new LongBitFormat(name(), previous, length, 0); } - long retrieve(long record) { - return (record & MASK) >>> OFFSET; + static String getUser(long permission) { + final int n = (int)USER.BITS.retrieve(permission); + return SerialNumberManager.INSTANCE.getUser(n); } - long combine(long bits, long record) { - return (record & ~MASK) | (bits << OFFSET); + static String getGroup(long permission) { + final int n = (int)GROUP.BITS.retrieve(permission); + return SerialNumberManager.INSTANCE.getGroup(n); + } + + static short getMode(long permission) { + return (short)MODE.BITS.retrieve(permission); } /** Encode the {@link PermissionStatus} to a long. */ @@ -62,12 +64,12 @@ public abstract class INodeWithAdditiona long permission = 0L; final int user = SerialNumberManager.INSTANCE.getUserSerialNumber( ps.getUserName()); - permission = USER.combine(user, permission); + permission = USER.BITS.combine(user, permission); final int group = SerialNumberManager.INSTANCE.getGroupSerialNumber( ps.getGroupName()); - permission = GROUP.combine(group, permission); + permission = GROUP.BITS.combine(group, permission); final int mode = ps.getPermission().toShort(); - permission = MODE.combine(mode, permission); + permission = MODE.BITS.combine(mode, permission); return permission; } } @@ -161,7 +163,7 @@ public abstract class INodeWithAdditiona } private final void updatePermissionStatus(PermissionStatusFormat f, long n) { - this.permission = f.combine(n, permission); + this.permission = f.BITS.combine(n, permission); } @Override @@ -169,9 +171,7 @@ public abstract class INodeWithAdditiona if (snapshotId != Snapshot.CURRENT_STATE_ID) { return getSnapshotINode(snapshotId).getUserName(); } - - int n = (int)PermissionStatusFormat.USER.retrieve(permission); - return SerialNumberManager.INSTANCE.getUser(n); + return PermissionStatusFormat.getUser(permission); } @Override @@ -185,9 +185,7 @@ public abstract class INodeWithAdditiona if (snapshotId != Snapshot.CURRENT_STATE_ID) { return getSnapshotINode(snapshotId).getGroupName(); } - - int n = (int)PermissionStatusFormat.GROUP.retrieve(permission); - return SerialNumberManager.INSTANCE.getGroup(n); + return PermissionStatusFormat.getGroup(permission); } @Override @@ -207,7 +205,7 @@ public abstract class INodeWithAdditiona @Override public final short getFsPermissionShort() { - return (short)PermissionStatusFormat.MODE.retrieve(permission); + return PermissionStatusFormat.getMode(permission); } @Override void setPermission(FsPermission permission) { @@ -317,8 +315,9 @@ public abstract class INodeWithAdditiona } protected <T extends Feature> T getFeature(Class<? extends Feature> clazz) { + Preconditions.checkArgument(clazz != null); for (Feature f : features) { - if (f.getClass() == clazz) { + if (clazz.isAssignableFrom(f.getClass())) { @SuppressWarnings("unchecked") T ret = (T) f; return ret; @@ -340,6 +339,30 @@ public abstract class INodeWithAdditiona addFeature(f); } + + @Override + XAttrFeature getXAttrFeature(int snapshotId) { + if (snapshotId != Snapshot.CURRENT_STATE_ID) { + return getSnapshotINode(snapshotId).getXAttrFeature(); + } + + return getFeature(XAttrFeature.class); + } + + @Override + public void removeXAttrFeature() { + XAttrFeature f = getXAttrFeature(); + Preconditions.checkNotNull(f); + removeFeature(f); + } + + @Override + public void addXAttrFeature(XAttrFeature f) { + XAttrFeature f1 = getXAttrFeature(); + Preconditions.checkState(f1 == null, "Duplicated XAttrFeature"); + + addFeature(f); + } public final Feature[] getFeatures() { return features; Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java Tue Aug 19 23:49:39 2014 @@ -27,7 +27,6 @@ import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.UnresolvedPathException; import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature; -import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; import com.google.common.base.Preconditions; @@ -46,6 +45,28 @@ public class INodesInPath { : Arrays.equals(HdfsConstants.DOT_SNAPSHOT_DIR_BYTES, pathComponent); } + static INodesInPath fromINode(INode inode) { + int depth = 0, index; + INode tmp = inode; + while (tmp != null) { + depth++; + tmp = tmp.getParent(); + } + final byte[][] path = new byte[depth][]; + final INode[] inodes = new INode[depth]; + final INodesInPath iip = new INodesInPath(path, depth); + tmp = inode; + index = depth; + while (tmp != null) { + index--; + path[index] = tmp.getKey(); + inodes[index] = tmp; + tmp = tmp.getParent(); + } + iip.setINodes(inodes); + return iip; + } + /** * Given some components, create a path name. * @param components The path components @@ -186,8 +207,7 @@ public class INodesInPath { final byte[] childName = components[count + 1]; // check if the next byte[] in components is for ".snapshot" - if (isDotSnapshotDir(childName) - && isDir && dir instanceof INodeDirectorySnapshottable) { + if (isDotSnapshotDir(childName) && isDir && dir.isSnapshottable()) { // skip the ".snapshot" in components count++; index++; @@ -200,8 +220,7 @@ public class INodesInPath { break; } // Resolve snapshot root - final Snapshot s = ((INodeDirectorySnapshottable)dir).getSnapshot( - components[count + 1]); + final Snapshot s = dir.getSnapshot(components[count + 1]); if (s == null) { //snapshot not found curNode = null; @@ -341,6 +360,11 @@ public class INodesInPath { private void addNode(INode node) { inodes[numNonNull++] = node; } + + private void setINodes(INode inodes[]) { + this.inodes = inodes; + this.numNonNull = this.inodes.length; + } void setINode(int i, INode inode) { inodes[i >= 0? i: inodes.length + i] = inode; @@ -417,4 +441,4 @@ public class INodesInPath { + ", this=" + toString(false)); } } -} \ No newline at end of file +} Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java Tue Aug 19 23:49:39 2014 @@ -82,7 +82,7 @@ public class ImageServlet extends HttpSe private static final String IMAGE_FILE_TYPE = "imageFile"; private static final Set<Long> currentlyDownloadingCheckpoints = - Collections.<Long>synchronizedSet(new HashSet<Long>()); + Collections.synchronizedSet(new HashSet<Long>()); @Override public void doGet(final HttpServletRequest request, Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java Tue Aug 19 23:49:39 2014 @@ -402,7 +402,6 @@ public class LeaseManager { /** * Get the list of inodes corresponding to valid leases. * @return list of inodes - * @throws UnresolvedLinkException */ Map<String, INodeFile> getINodesUnderConstruction() { Map<String, INodeFile> inodes = new TreeMap<String, INodeFile>(); Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LogsPurgeable.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LogsPurgeable.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LogsPurgeable.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LogsPurgeable.java Tue Aug 19 23:49:39 2014 @@ -42,7 +42,6 @@ interface LogsPurgeable { * * @param fromTxId the first transaction id we want to read * @param inProgressOk whether or not in-progress streams should be returned - * @return a list of streams * @throws IOException if the underlying storage has an error or is otherwise * inaccessible */ Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java Tue Aug 19 23:49:39 2014 @@ -77,7 +77,8 @@ public class NNStorage extends Storage i IMAGE_ROLLBACK("fsimage_rollback"), EDITS_NEW ("edits.new"), // from "old" pre-HDFS-1073 format EDITS_INPROGRESS ("edits_inprogress"), - EDITS_TMP ("edits_tmp"); + EDITS_TMP ("edits_tmp"), + IMAGE_LEGACY_OIV ("fsimage_legacy_oiv"); // For pre-PB format private String fileName = null; private NameNodeFile(String name) { this.fileName = name; } @@ -425,8 +426,7 @@ public class NNStorage extends Storage i /** * Write last checkpoint time into a separate file. - * - * @param sd + * @param sd storage directory * @throws IOException */ void writeTransactionIdFile(StorageDirectory sd, long txid) throws IOException { @@ -694,6 +694,10 @@ public class NNStorage extends Storage i return getNameNodeFileName(NameNodeFile.IMAGE_ROLLBACK, txid); } + public static String getLegacyOIVImageFileName(long txid) { + return getNameNodeFileName(NameNodeFile.IMAGE_LEGACY_OIV, txid); + } + private static String getNameNodeFileName(NameNodeFile nnf, long txid) { return String.format("%s_%019d", nnf.getName(), txid); } @@ -832,7 +836,7 @@ public class NNStorage extends Storage i */ void processStartupOptionsForUpgrade(StartupOption startOpt, int layoutVersion) throws IOException { - if (startOpt == StartupOption.UPGRADE) { + if (startOpt == StartupOption.UPGRADE || startOpt == StartupOption.UPGRADEONLY) { // If upgrade from a release that does not support federation, // if clusterId is provided in the startupOptions use it. // Else generate a new cluster ID Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java Tue Aug 19 23:49:39 2014 @@ -18,11 +18,13 @@ package org.apache.hadoop.hdfs.server.namenode; import java.io.File; +import java.io.FilenameFilter; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.EnumSet; +import java.util.Iterator; import java.util.List; import java.util.TreeSet; @@ -233,4 +235,58 @@ public class NNStorageRetentionManager { } } } + + /** + * Delete old OIV fsimages. Since the target dir is not a full blown + * storage directory, we simply list and keep the latest ones. For the + * same reason, no storage inspector is used. + */ + void purgeOldLegacyOIVImages(String dir, long txid) { + File oivImageDir = new File(dir); + final String oivImagePrefix = NameNodeFile.IMAGE_LEGACY_OIV.getName(); + String filesInStorage[]; + + // Get the listing + filesInStorage = oivImageDir.list(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return name.matches(oivImagePrefix + "_(\\d+)"); + } + }); + + // Check whether there is any work to do. + if (filesInStorage.length <= numCheckpointsToRetain) { + return; + } + + // Create a sorted list of txids from the file names. + TreeSet<Long> sortedTxIds = new TreeSet<Long>(); + for (String fName : filesInStorage) { + // Extract the transaction id from the file name. + long fTxId; + try { + fTxId = Long.parseLong(fName.substring(oivImagePrefix.length() + 1)); + } catch (NumberFormatException nfe) { + // This should not happen since we have already filtered it. + // Log and continue. + LOG.warn("Invalid file name. Skipping " + fName); + continue; + } + sortedTxIds.add(Long.valueOf(fTxId)); + } + + int numFilesToDelete = sortedTxIds.size() - numCheckpointsToRetain; + Iterator<Long> iter = sortedTxIds.iterator(); + while (numFilesToDelete > 0 && iter.hasNext()) { + long txIdVal = iter.next().longValue(); + String fileName = NNStorage.getLegacyOIVImageFileName(txIdVal); + LOG.info("Deleting " + fileName); + File fileToDelete = new File(oivImageDir, fileName); + if (!fileToDelete.delete()) { + // deletion failed. + LOG.warn("Failed to delete image file: " + fileToDelete); + } + numFilesToDelete--; + } + } } Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java Tue Aug 19 23:49:39 2014 @@ -175,6 +175,8 @@ public class NameNode implements NameNod DFS_NAMENODE_SERVICE_RPC_BIND_HOST_KEY, DFS_NAMENODE_HTTP_ADDRESS_KEY, DFS_NAMENODE_HTTPS_ADDRESS_KEY, + DFS_NAMENODE_HTTP_BIND_HOST_KEY, + DFS_NAMENODE_HTTPS_BIND_HOST_KEY, DFS_NAMENODE_KEYTAB_FILE_KEY, DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY, DFS_NAMENODE_SECONDARY_HTTPS_ADDRESS_KEY, @@ -199,25 +201,31 @@ public class NameNode implements NameNod }; private static final String USAGE = "Usage: java NameNode [" - + StartupOption.BACKUP.getName() + "] | [" - + StartupOption.CHECKPOINT.getName() + "] | [" + + StartupOption.BACKUP.getName() + "] | \n\t[" + + StartupOption.CHECKPOINT.getName() + "] | \n\t[" + StartupOption.FORMAT.getName() + " [" + StartupOption.CLUSTERID.getName() + " cid ] [" + StartupOption.FORCE.getName() + "] [" - + StartupOption.NONINTERACTIVE.getName() + "] ] | [" + + StartupOption.NONINTERACTIVE.getName() + "] ] | \n\t[" + StartupOption.UPGRADE.getName() + " [" + StartupOption.CLUSTERID.getName() + " cid]" + - " [" + StartupOption.RENAMERESERVED.getName() + "<k-v pairs>] ] | [" - + StartupOption.ROLLBACK.getName() + "] | [" + " [" + StartupOption.RENAMERESERVED.getName() + "<k-v pairs>] ] | \n\t[" + + StartupOption.UPGRADEONLY.getName() + + " [" + StartupOption.CLUSTERID.getName() + " cid]" + + " [" + StartupOption.RENAMERESERVED.getName() + "<k-v pairs>] ] | \n\t[" + + StartupOption.ROLLBACK.getName() + "] | \n\t[" + StartupOption.ROLLINGUPGRADE.getName() + " <" + RollingUpgradeStartupOption.DOWNGRADE.name().toLowerCase() + "|" - + RollingUpgradeStartupOption.ROLLBACK.name().toLowerCase() + "> ] | [" - + StartupOption.FINALIZE.getName() + "] | [" - + StartupOption.IMPORT.getName() + "] | [" - + StartupOption.INITIALIZESHAREDEDITS.getName() + "] | [" - + StartupOption.BOOTSTRAPSTANDBY.getName() + "] | [" - + StartupOption.RECOVER.getName() + " [ " + StartupOption.FORCE.getName() - + " ] ]"; + + RollingUpgradeStartupOption.ROLLBACK.name().toLowerCase() + "> ] | \n\t[" + + StartupOption.FINALIZE.getName() + "] | \n\t[" + + StartupOption.IMPORT.getName() + "] | \n\t[" + + StartupOption.INITIALIZESHAREDEDITS.getName() + "] | \n\t[" + + StartupOption.BOOTSTRAPSTANDBY.getName() + "] | \n\t[" + + StartupOption.RECOVER.getName() + " [ " + + StartupOption.FORCE.getName() + "] ] | \n\t[" + + StartupOption.METADATAVERSION.getName() + " ] " + + " ]"; + public long getProtocolVersion(String protocol, long clientVersion) throws IOException { @@ -271,10 +279,11 @@ public class NameNode implements NameNod private JvmPauseMonitor pauseMonitor; private ObjectName nameNodeStatusBeanName; /** - * The service name of the delegation token issued by the namenode. It is - * the name service id in HA mode, or the rpc address in non-HA mode. + * The namenode address that clients will use to access this namenode + * or the name service. For HA configurations using logical URI, it + * will be the logical address. */ - private String tokenServiceName; + private String clientNamenodeAddress; /** Format a new filesystem. Destroys any filesystem that may already * exist at this location. **/ @@ -317,7 +326,54 @@ public class NameNode implements NameNod * * @return The name service id in HA-mode, or the rpc address in non-HA mode */ - public String getTokenServiceName() { return tokenServiceName; } + public String getTokenServiceName() { + return getClientNamenodeAddress(); + } + + /** + * Set the namenode address that will be used by clients to access this + * namenode or name service. This needs to be called before the config + * is overriden. + */ + public void setClientNamenodeAddress(Configuration conf) { + String nnAddr = conf.get(FS_DEFAULT_NAME_KEY); + if (nnAddr == null) { + // default fs is not set. + clientNamenodeAddress = null; + return; + } + + LOG.info(FS_DEFAULT_NAME_KEY + " is " + nnAddr); + URI nnUri = URI.create(nnAddr); + + String nnHost = nnUri.getHost(); + if (nnHost == null) { + clientNamenodeAddress = null; + return; + } + + if (DFSUtil.getNameServiceIds(conf).contains(nnHost)) { + // host name is logical + clientNamenodeAddress = nnHost; + } else if (nnUri.getPort() > 0) { + // physical address with a valid port + clientNamenodeAddress = nnUri.getAuthority(); + } else { + // the port is missing or 0. Figure out real bind address later. + clientNamenodeAddress = null; + return; + } + LOG.info("Clients are to use " + clientNamenodeAddress + " to access" + + " this namenode/service."); + } + + /** + * Get the namenode address to be used by clients. + * @return nn address + */ + public String getClientNamenodeAddress() { + return clientNamenodeAddress; + } public static InetSocketAddress getAddress(String address) { return NetUtils.createSocketAddr(address, DEFAULT_PORT); @@ -356,8 +412,6 @@ public class NameNode implements NameNod /** - * TODO:FEDERATION - * @param filesystemURI * @return address of file system */ public static InetSocketAddress getAddress(URI filesystemURI) { @@ -446,6 +500,29 @@ public class NameNode implements NameNod return getHttpAddress(conf); } + /** + * HTTP server address for binding the endpoint. This method is + * for use by the NameNode and its derivatives. It may return + * a different address than the one that should be used by clients to + * connect to the NameNode. See + * {@link DFSConfigKeys#DFS_NAMENODE_HTTP_BIND_HOST_KEY} + * + * @param conf + * @return + */ + protected InetSocketAddress getHttpServerBindAddress(Configuration conf) { + InetSocketAddress bindAddress = getHttpServerAddress(conf); + + // If DFS_NAMENODE_HTTP_BIND_HOST_KEY exists then it overrides the + // host name portion of DFS_NAMENODE_HTTP_ADDRESS_KEY. + final String bindHost = conf.getTrimmed(DFS_NAMENODE_HTTP_BIND_HOST_KEY); + if (bindHost != null && !bindHost.isEmpty()) { + bindAddress = new InetSocketAddress(bindHost, bindAddress.getPort()); + } + + return bindAddress; + } + /** @return the NameNode HTTP address. */ public static InetSocketAddress getHttpAddress(Configuration conf) { return NetUtils.createSocketAddr( @@ -512,9 +589,14 @@ public class NameNode implements NameNod loadNamesystem(conf); rpcServer = createRpcServer(conf); - final String nsId = getNameServiceId(conf); - tokenServiceName = HAUtil.isHAEnabled(conf, nsId) ? nsId : NetUtils - .getHostPortString(rpcServer.getRpcAddress()); + if (clientNamenodeAddress == null) { + // This is expected for MiniDFSCluster. Set it now using + // the RPC server's bind address. + clientNamenodeAddress = + NetUtils.getHostPortString(rpcServer.getRpcAddress()); + LOG.info("Clients are to use " + clientNamenodeAddress + " to access" + + " this namenode/service."); + } if (NamenodeRole.NAMENODE == role) { httpServer.setNameNodeAddress(getNameNodeAddress()); httpServer.setFSImage(getFSImage()); @@ -522,7 +604,8 @@ public class NameNode implements NameNod pauseMonitor = new JvmPauseMonitor(conf); pauseMonitor.start(); - + metrics.getJvmMetrics().setPauseMonitor(pauseMonitor); + startCommonServices(conf); } @@ -610,7 +693,7 @@ public class NameNode implements NameNod } private void startHttpServer(final Configuration conf) throws IOException { - httpServer = new NameNodeHttpServer(conf, this, getHttpServerAddress(conf)); + httpServer = new NameNodeHttpServer(conf, this, getHttpServerBindAddress(conf)); httpServer.start(); httpServer.setStartupProgress(startupProgress); } @@ -633,6 +716,7 @@ public class NameNode implements NameNod * <li>{@link StartupOption#BACKUP BACKUP} - start backup node</li> * <li>{@link StartupOption#CHECKPOINT CHECKPOINT} - start checkpoint node</li> * <li>{@link StartupOption#UPGRADE UPGRADE} - start the cluster + * <li>{@link StartupOption#UPGRADEONLY UPGRADEONLY} - upgrade the cluster * upgrade and create a snapshot of the current file system state</li> * <li>{@link StartupOption#RECOVER RECOVERY} - recover name node * metadata</li> @@ -660,6 +744,7 @@ public class NameNode implements NameNod throws IOException { this.conf = conf; this.role = role; + setClientNamenodeAddress(conf); String nsId = getNameServiceId(conf); String namenodeId = HAUtil.getNameNodeId(conf, nsId); this.haEnabled = HAUtil.isHAEnabled(conf, nsId); @@ -686,7 +771,8 @@ public class NameNode implements NameNod } protected HAState createHAState(StartupOption startOpt) { - if (!haEnabled || startOpt == StartupOption.UPGRADE) { + if (!haEnabled || startOpt == StartupOption.UPGRADE + || startOpt == StartupOption.UPGRADEONLY) { return ACTIVE_STATE; } else { return STANDBY_STATE; @@ -753,7 +839,7 @@ public class NameNode implements NameNod /** get FSImage */ @VisibleForTesting public FSImage getFSImage() { - return namesystem.dir.fsImage; + return namesystem.getFSImage(); } /** @@ -800,8 +886,8 @@ public class NameNode implements NameNod * Interactively confirm that formatting is desired * for each existing directory and format them. * - * @param conf - * @param force + * @param conf configuration to use + * @param force if true, format regardless of whether dirs exist * @return true if formatting was aborted, false otherwise * @throws IOException */ @@ -1063,7 +1149,7 @@ public class NameNode implements NameNod return true; } } - nsys.dir.fsImage.doRollback(nsys); + nsys.getFSImage().doRollback(nsys); return false; } @@ -1117,8 +1203,10 @@ public class NameNode implements NameNod startOpt = StartupOption.BACKUP; } else if (StartupOption.CHECKPOINT.getName().equalsIgnoreCase(cmd)) { startOpt = StartupOption.CHECKPOINT; - } else if (StartupOption.UPGRADE.getName().equalsIgnoreCase(cmd)) { - startOpt = StartupOption.UPGRADE; + } else if (StartupOption.UPGRADE.getName().equalsIgnoreCase(cmd) + || StartupOption.UPGRADEONLY.getName().equalsIgnoreCase(cmd)) { + startOpt = StartupOption.UPGRADE.getName().equalsIgnoreCase(cmd) ? + StartupOption.UPGRADE : StartupOption.UPGRADEONLY; /* Can be followed by CLUSTERID with a required parameter or * RENAMERESERVED with an optional parameter */ @@ -1188,6 +1276,8 @@ public class NameNode implements NameNod "can't understand option \"" + args[i] + "\""); } } + } else if (StartupOption.METADATAVERSION.getName().equalsIgnoreCase(cmd)) { + startOpt = StartupOption.METADATAVERSION; } else { return null; } @@ -1240,6 +1330,21 @@ public class NameNode implements NameNod } } + /** + * Verify that configured directories exist, then print the metadata versions + * of the software and the image. + * + * @param conf configuration to use + * @throws IOException + */ + private static boolean printMetadataVersion(Configuration conf) + throws IOException { + final FSImage fsImage = new FSImage(conf); + final FSNamesystem fs = new FSNamesystem(conf, fsImage, false); + return fsImage.recoverTransitionRead( + StartupOption.METADATAVERSION, fs, null); + } + public static NameNode createNameNode(String argv[], Configuration conf) throws IOException { LOG.info("createNameNode " + Arrays.asList(argv)); @@ -1304,6 +1409,17 @@ public class NameNode implements NameNod NameNode.doRecovery(startOpt, conf); return null; } + case METADATAVERSION: { + printMetadataVersion(conf); + terminate(0); + return null; // avoid javac warning + } + case UPGRADEONLY: { + DefaultMetricsSystem.initialize("NameNode"); + new NameNode(conf); + terminate(0); + return null; + } default: { DefaultMetricsSystem.initialize("NameNode"); return new NameNode(conf); @@ -1600,7 +1716,11 @@ public class NameNode implements NameNod public boolean isStandbyState() { return (state.equals(STANDBY_STATE)); } - + + public boolean isActiveState() { + return (state.equals(ACTIVE_STATE)); + } + /** * Check that a request to change this node's HA state is valid. * In particular, verifies that, if auto failover is enabled, non-forced
