Author: wheat9 Date: Tue Jun 24 20:31:18 2014 New Revision: 1605180 URL: http://svn.apache.org/r1605180 Log: HDFS-6593. Merge r1605169 from trunk.
Added: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java - copied unchanged from r1605169, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1605180&r1=1605179&r2=1605180&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Tue Jun 24 20:31:18 2014 @@ -219,6 +219,9 @@ Release 2.5.0 - UNRELEASED HDFS-6430. HTTPFS - Implement XAttr support. (Yi Liu via tucu) + HDFS-6593. Move SnapshotDiffInfo out of INodeDirectorySnapshottable. + (Jing Zhao via wheat9) + OPTIMIZATIONS HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn) Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java?rev=1605180&r1=1605179&r2=1605180&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java Tue Jun 24 20:31:18 2014 @@ -23,16 +23,14 @@ import java.util.List; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSUtil; -import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable.SnapshotDiffInfo; import com.google.common.base.Objects; /** * This class represents to end users the difference between two snapshots of * the same directory, or the difference between a snapshot of the directory and - * its current state. Instead of capturing all the details of the diff, which - * is stored in {@link SnapshotDiffInfo}, this class only lists where the - * changes happened and their types. + * its current state. Instead of capturing all the details of the diff, this + * class only lists where the changes happened and their types. */ public class SnapshotDiffReport { private final static String LINE_SEPARATOR = System.getProperty( Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1605180&r1=1605179&r2=1605180&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Tue Jun 24 20:31:18 2014 @@ -87,7 +87,18 @@ import static org.apache.hadoop.hdfs.DFS import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SUPPORT_APPEND_KEY; import static org.apache.hadoop.util.Time.now; -import java.io.*; +import java.io.BufferedWriter; +import java.io.ByteArrayInputStream; +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.io.StringWriter; import java.lang.management.ManagementFactory; import java.net.InetAddress; import java.net.URI; @@ -173,7 +184,6 @@ import org.apache.hadoop.hdfs.protocol.R import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo; import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; -import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; import org.apache.hadoop.hdfs.protocol.datatransfer.ReplaceDatanodeOnFailure; import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager; @@ -211,7 +221,6 @@ import org.apache.hadoop.hdfs.server.nam import org.apache.hadoop.hdfs.server.namenode.metrics.FSNamesystemMBean; import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics; import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable; -import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable.SnapshotDiffInfo; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotManager; import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase; @@ -7483,7 +7492,7 @@ public class FSNamesystem implements Nam */ SnapshotDiffReport getSnapshotDiffReport(String path, String fromSnapshot, String toSnapshot) throws IOException { - SnapshotDiffInfo diffs = null; + SnapshotDiffReport diffs; checkOperation(OperationCategory.READ); final FSPermissionChecker pc = getPermissionChecker(); readLock(); @@ -7497,13 +7506,11 @@ public class FSNamesystem implements Nam } finally { readUnlock(); } - + if (auditLog.isInfoEnabled() && isExternalInvocation()) { logAuditEvent(true, "computeSnapshotDiff", null, null, null); } - return diffs != null ? diffs.generateReport() : new SnapshotDiffReport( - path, fromSnapshot, toSnapshot, - Collections.<DiffReportEntry> emptyList()); + return diffs; } private void checkSubtreeReadPermission(final FSPermissionChecker pc, Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java?rev=1605180&r1=1605179&r2=1605180&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java Tue Jun 24 20:31:18 2014 @@ -20,7 +20,6 @@ package org.apache.hadoop.hdfs.server.na import java.io.DataOutput; import java.io.IOException; import java.util.ArrayDeque; -import java.util.ArrayList; import java.util.Deque; import java.util.HashMap; import java.util.Iterator; @@ -29,8 +28,6 @@ import java.util.Map; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.protocol.QuotaExceededException; -import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry; -import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType; import org.apache.hadoop.hdfs.server.namenode.Content; import org.apache.hadoop.hdfs.server.namenode.ContentSummaryComputationContext; import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization; @@ -41,7 +38,6 @@ import org.apache.hadoop.hdfs.server.nam import org.apache.hadoop.hdfs.server.namenode.INodeFile; import org.apache.hadoop.hdfs.server.namenode.INodeReference; import org.apache.hadoop.hdfs.server.namenode.Quota; -import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable.SnapshotDiffInfo.RenameEntry; import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat.ReferenceMap; import org.apache.hadoop.hdfs.util.Diff; import org.apache.hadoop.hdfs.util.Diff.Container; @@ -161,43 +157,6 @@ public class DirectoryWithSnapshotFeatur } } } - - /** - * Interpret the diff and generate a list of {@link DiffReportEntry}. - * @param parentPath The relative path of the parent. - * @param fromEarlier True indicates {@code diff=later-earlier}, - * False indicates {@code diff=earlier-later} - * @return A list of {@link DiffReportEntry} as the diff report. - */ - public List<DiffReportEntry> generateReport(byte[][] parentPath, - boolean fromEarlier, Map<Long, RenameEntry> renameMap) { - List<DiffReportEntry> list = new ArrayList<DiffReportEntry>(); - List<INode> created = getList(ListType.CREATED); - List<INode> deleted = getList(ListType.DELETED); - byte[][] fullPath = new byte[parentPath.length + 1][]; - System.arraycopy(parentPath, 0, fullPath, 0, parentPath.length); - for (INode cnode : created) { - RenameEntry entry = renameMap.get(cnode.getId()); - if (entry == null || !entry.isRename()) { - fullPath[fullPath.length - 1] = cnode.getLocalNameBytes(); - list.add(new DiffReportEntry(fromEarlier ? DiffType.CREATE - : DiffType.DELETE, fullPath)); - } - } - for (INode dnode : deleted) { - RenameEntry entry = renameMap.get(dnode.getId()); - if (entry != null && entry.isRename()) { - list.add(new DiffReportEntry(DiffType.RENAME, - fromEarlier ? entry.getSourcePath() : entry.getTargetPath(), - fromEarlier ? entry.getTargetPath() : entry.getSourcePath())); - } else { - fullPath[fullPath.length - 1] = dnode.getLocalNameBytes(); - list.add(new DiffReportEntry(fromEarlier ? DiffType.DELETE - : DiffType.CREATE, fullPath)); - } - } - return list; - } } /** Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java?rev=1605180&r1=1605179&r2=1605180&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java Tue Jun 24 20:31:18 2014 @@ -21,22 +21,14 @@ import java.io.IOException; import java.io.PrintWriter; import java.util.ArrayList; import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; import java.util.Iterator; import java.util.LinkedList; import java.util.List; -import java.util.Map; -import java.util.SortedMap; -import java.util.TreeMap; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.protocol.QuotaExceededException; -import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; -import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry; -import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType; import org.apache.hadoop.hdfs.protocol.SnapshotException; import org.apache.hadoop.hdfs.server.namenode.Content; import org.apache.hadoop.hdfs.server.namenode.ContentSummaryComputationContext; @@ -56,7 +48,6 @@ import org.apache.hadoop.util.Time; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; -import com.google.common.primitives.SignedBytes; /** * Directories where taking snapshots is allowed. @@ -79,164 +70,6 @@ public class INodeDirectorySnapshottable } return (INodeDirectorySnapshottable)dir; } - - /** - * A class describing the difference between snapshots of a snapshottable - * directory. - */ - public static class SnapshotDiffInfo { - /** Compare two inodes based on their full names */ - public static final Comparator<INode> INODE_COMPARATOR = - new Comparator<INode>() { - @Override - public int compare(INode left, INode right) { - if (left == null) { - return right == null ? 0 : -1; - } else { - if (right == null) { - return 1; - } else { - int cmp = compare(left.getParent(), right.getParent()); - return cmp == 0 ? SignedBytes.lexicographicalComparator().compare( - left.getLocalNameBytes(), right.getLocalNameBytes()) : cmp; - } - } - } - }; - - static class RenameEntry { - private byte[][] sourcePath; - private byte[][] targetPath; - - void setSource(INode source, byte[][] sourceParentPath) { - Preconditions.checkState(sourcePath == null); - sourcePath = new byte[sourceParentPath.length + 1][]; - System.arraycopy(sourceParentPath, 0, sourcePath, 0, - sourceParentPath.length); - sourcePath[sourcePath.length - 1] = source.getLocalNameBytes(); - } - - void setTarget(INode target, byte[][] targetParentPath) { - targetPath = new byte[targetParentPath.length + 1][]; - System.arraycopy(targetParentPath, 0, targetPath, 0, - targetParentPath.length); - targetPath[targetPath.length - 1] = target.getLocalNameBytes(); - } - - void setTarget(byte[][] targetPath) { - this.targetPath = targetPath; - } - - boolean isRename() { - return sourcePath != null && targetPath != null; - } - - byte[][] getSourcePath() { - return sourcePath; - } - - byte[][] getTargetPath() { - return targetPath; - } - } - - /** The root directory of the snapshots */ - private final INodeDirectorySnapshottable snapshotRoot; - /** The starting point of the difference */ - private final Snapshot from; - /** The end point of the difference */ - private final Snapshot to; - /** - * A map recording modified INodeFile and INodeDirectory and their relative - * path corresponding to the snapshot root. Sorted based on their names. - */ - private final SortedMap<INode, byte[][]> diffMap = - new TreeMap<INode, byte[][]>(INODE_COMPARATOR); - /** - * A map capturing the detailed difference about file creation/deletion. - * Each key indicates a directory whose children have been changed between - * the two snapshots, while its associated value is a {@link ChildrenDiff} - * storing the changes (creation/deletion) happened to the children (files). - */ - private final Map<INodeDirectory, ChildrenDiff> dirDiffMap = - new HashMap<INodeDirectory, ChildrenDiff>(); - - private final Map<Long, RenameEntry> renameMap = - new HashMap<Long, RenameEntry>(); - - SnapshotDiffInfo(INodeDirectorySnapshottable snapshotRoot, Snapshot start, - Snapshot end) { - this.snapshotRoot = snapshotRoot; - this.from = start; - this.to = end; - } - - /** Add a dir-diff pair */ - private void addDirDiff(INodeDirectory dir, byte[][] relativePath, - ChildrenDiff diff) { - dirDiffMap.put(dir, diff); - diffMap.put(dir, relativePath); - // detect rename - for (INode created : diff.getList(ListType.CREATED)) { - if (created.isReference()) { - RenameEntry entry = getEntry(created.getId()); - if (entry.getTargetPath() == null) { - entry.setTarget(created, relativePath); - } - } - } - for (INode deleted : diff.getList(ListType.DELETED)) { - if (deleted instanceof INodeReference.WithName) { - RenameEntry entry = getEntry(deleted.getId()); - entry.setSource(deleted, relativePath); - } - } - } - - private RenameEntry getEntry(long inodeId) { - RenameEntry entry = renameMap.get(inodeId); - if (entry == null) { - entry = new RenameEntry(); - renameMap.put(inodeId, entry); - } - return entry; - } - - private void setRenameTarget(long inodeId, byte[][] path) { - getEntry(inodeId).setTarget(path); - } - - /** Add a modified file */ - private void addFileDiff(INodeFile file, byte[][] relativePath) { - diffMap.put(file, relativePath); - } - - /** @return True if {@link #from} is earlier than {@link #to} */ - private boolean isFromEarlier() { - return Snapshot.ID_COMPARATOR.compare(from, to) < 0; - } - - /** - * Generate a {@link SnapshotDiffReport} based on detailed diff information. - * @return A {@link SnapshotDiffReport} describing the difference - */ - public SnapshotDiffReport generateReport() { - List<DiffReportEntry> diffReportList = new ArrayList<DiffReportEntry>(); - for (INode node : diffMap.keySet()) { - diffReportList.add(new DiffReportEntry(DiffType.MODIFY, diffMap - .get(node), null)); - if (node.isDirectory()) { - ChildrenDiff dirDiff = dirDiffMap.get(node); - List<DiffReportEntry> subList = dirDiff.generateReport( - diffMap.get(node), isFromEarlier(), renameMap); - diffReportList.addAll(subList); - } - } - return new SnapshotDiffReport(snapshotRoot.getFullPathName(), - Snapshot.getSnapshotName(from), Snapshot.getSnapshotName(to), - diffReportList); - } - } /** * Snapshots of this directory in ascending order of snapshot names. @@ -496,9 +329,9 @@ public class INodeDirectorySnapshottable private void computeDiffRecursively(INode node, List<byte[]> parentPath, SnapshotDiffInfo diffReport) { final Snapshot earlierSnapshot = diffReport.isFromEarlier() ? - diffReport.from : diffReport.to; + diffReport.getFrom() : diffReport.getTo(); final Snapshot laterSnapshot = diffReport.isFromEarlier() ? - diffReport.to : diffReport.from; + diffReport.getTo() : diffReport.getFrom(); byte[][] relativePath = parentPath.toArray(new byte[parentPath.size()][]); if (node.isDirectory()) { final ChildrenDiff diff = new ChildrenDiff(); Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java?rev=1605180&r1=1605179&r2=1605180&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java Tue Jun 24 20:31:18 2014 @@ -30,9 +30,11 @@ import java.util.concurrent.atomic.Atomi import javax.management.ObjectName; import org.apache.hadoop.hdfs.DFSUtil; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.SnapshotException; import org.apache.hadoop.hdfs.protocol.SnapshotInfo; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry; import org.apache.hadoop.hdfs.server.namenode.FSDirectory; import org.apache.hadoop.hdfs.server.namenode.FSImageFormat; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; @@ -40,7 +42,6 @@ import org.apache.hadoop.hdfs.server.nam import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo; import org.apache.hadoop.hdfs.server.namenode.INodeDirectory; import org.apache.hadoop.hdfs.server.namenode.INodesInPath; -import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable.SnapshotDiffInfo; import org.apache.hadoop.metrics2.util.MBeans; /** @@ -361,12 +362,13 @@ public class SnapshotManager implements * Compute the difference between two snapshots of a directory, or between a * snapshot of the directory and its current tree. */ - public SnapshotDiffInfo diff(final String path, final String from, + public SnapshotDiffReport diff(final String path, final String from, final String to) throws IOException { if ((from == null || from.isEmpty()) && (to == null || to.isEmpty())) { // both fromSnapshot and toSnapshot indicate the current tree - return null; + return new SnapshotDiffReport(path, from, to, + Collections.<DiffReportEntry> emptyList()); } // Find the source root directory path where the snapshots were taken. @@ -374,8 +376,10 @@ public class SnapshotManager implements INodesInPath inodesInPath = fsdir.getINodesInPath4Write(path.toString()); final INodeDirectorySnapshottable snapshotRoot = INodeDirectorySnapshottable .valueOf(inodesInPath.getLastINode(), path); - - return snapshotRoot.computeDiff(from, to); + + final SnapshotDiffInfo diffs = snapshotRoot.computeDiff(from, to); + return diffs != null ? diffs.generateReport() : new SnapshotDiffReport( + path, from, to, Collections.<DiffReportEntry> emptyList()); } public void clearSnapshottableDirs() {