Repository: hadoop Updated Branches: refs/heads/trunk f5fe35e29 -> 26971e52a
HADOOP-11852. Disable symlinks in trunk. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/26971e52 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/26971e52 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/26971e52 Branch: refs/heads/trunk Commit: 26971e52ae65590e618a23621be244e588845adc Parents: f5fe35e Author: Andrew Wang <w...@apache.org> Authored: Thu Apr 23 11:47:01 2015 -0700 Committer: Andrew Wang <w...@apache.org> Committed: Thu Apr 23 11:47:01 2015 -0700 ---------------------------------------------------------------------- hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++ .../java/org/apache/hadoop/fs/FSLinkResolver.java | 4 ++++ .../main/java/org/apache/hadoop/fs/FileContext.java | 4 ++++ .../main/java/org/apache/hadoop/fs/FileSystem.java | 15 +++++++++++++++ .../org/apache/hadoop/fs/FileSystemLinkResolver.java | 4 ++++ .../org/apache/hadoop/fs/RawLocalFileSystem.java | 4 ++++ .../java/org/apache/hadoop/fs/SymlinkBaseTest.java | 4 ++++ .../apache/hadoop/fs/TestFileContextResolveAfs.java | 4 +++- .../src/test/java/org/apache/hadoop/fs/TestStat.java | 4 +++- .../apache/hadoop/hdfs/DistributedFileSystem.java | 4 ++++ .../hadoop/hdfs/server/namenode/FSEditLogLoader.java | 4 ++++ .../hadoop/hdfs/server/namenode/FSImageFormat.java | 5 +++++ .../hadoop/hdfs/server/namenode/FSNamesystem.java | 4 ++++ .../java/org/apache/hadoop/hdfs/MiniDFSCluster.java | 3 +++ .../hadoop/hdfs/server/namenode/TestINodeFile.java | 4 ++++ 15 files changed, 67 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/26971e52/hadoop-common-project/hadoop-common/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index db1425f..f232e04 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -201,6 +201,8 @@ Trunk (Unreleased) HADOOP-11850. Typos in hadoop-common java docs. (Surendra Singh Lilhore via jghoman) + HADOOP-11852. Disable symlinks in trunk. + BUG FIXES HADOOP-11473. test-patch says "-1 overall" even when all checks are +1 http://git-wip-us.apache.org/repos/asf/hadoop/blob/26971e52/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSLinkResolver.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSLinkResolver.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSLinkResolver.java index 831d4ca..ffe4b34 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSLinkResolver.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSLinkResolver.java @@ -95,6 +95,10 @@ public abstract class FSLinkResolver<T> { + " and symlink resolution is disabled (" + CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY + ").", e); } + if (!FileSystem.areSymlinksEnabled()) { + throw new IOException("Symlink resolution is disabled in" + + " this version of Hadoop."); + } if (count++ > FsConstants.MAX_PATH_LINKS) { throw new IOException("Possible cyclic loop while " + "following symbolic link " + path); http://git-wip-us.apache.org/repos/asf/hadoop/blob/26971e52/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java index 0b5863b..ea3f896 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java @@ -1431,11 +1431,15 @@ public class FileContext { * <code>target</code> or <code>link</code> is not supported * @throws IOException If an I/O error occurred */ + @SuppressWarnings("deprecation") public void createSymlink(final Path target, final Path link, final boolean createParent) throws AccessControlException, FileAlreadyExistsException, FileNotFoundException, ParentNotDirectoryException, UnsupportedFileSystemException, IOException { + if (!FileSystem.areSymlinksEnabled()) { + throw new UnsupportedOperationException("Symlinks not supported"); + } final Path nonRelLink = fixRelativePart(link); new FSLinkResolver<Void>() { @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/26971e52/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java index 79b7403..b508727 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -3296,4 +3296,19 @@ public abstract class FileSystem extends Configured implements Closeable { ": " + pair.getValue()); } } + + // Symlinks are temporarily disabled - see HADOOP-10020 and HADOOP-10052 + private static boolean symlinksEnabled = false; + + private static Configuration conf = null; + + @VisibleForTesting + public static boolean areSymlinksEnabled() { + return symlinksEnabled; + } + + @VisibleForTesting + public static void enableSymlinks() { + symlinksEnabled = true; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/26971e52/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemLinkResolver.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemLinkResolver.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemLinkResolver.java index c01d41f..7eec0eb 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemLinkResolver.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemLinkResolver.java @@ -87,6 +87,10 @@ public abstract class FileSystemLinkResolver<T> { + CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY + ").", e); } + if (!FileSystem.areSymlinksEnabled()) { + throw new IOException("Symlink resolution is disabled in" + + " this version of Hadoop."); + } if (count++ > FsConstants.MAX_PATH_LINKS) { throw new IOException("Possible cyclic loop while " + "following symbolic link " + path); http://git-wip-us.apache.org/repos/asf/hadoop/blob/26971e52/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java index 52623b8..56dd7ad 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java @@ -775,9 +775,13 @@ public class RawLocalFileSystem extends FileSystem { return true; } + @SuppressWarnings("deprecation") @Override public void createSymlink(Path target, Path link, boolean createParent) throws IOException { + if (!FileSystem.areSymlinksEnabled()) { + throw new UnsupportedOperationException("Symlinks not supported"); + } final String targetScheme = target.toUri().getScheme(); if (targetScheme != null && !"file".equals(targetScheme)) { throw new IOException("Unable to create symlink to non-local file "+ http://git-wip-us.apache.org/repos/asf/hadoop/blob/26971e52/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/SymlinkBaseTest.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/SymlinkBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/SymlinkBaseTest.java index 9fe2cd7..4d6485d 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/SymlinkBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/SymlinkBaseTest.java @@ -36,6 +36,10 @@ import org.junit.After; * Base test for symbolic links */ public abstract class SymlinkBaseTest { + // Re-enable symlinks for tests, see HADOOP-10020 and HADOOP-10052 + static { + FileSystem.enableSymlinks(); + } static final long seed = 0xDEADBEEFL; static final int blockSize = 8192; static final int fileSize = 16384; http://git-wip-us.apache.org/repos/asf/hadoop/blob/26971e52/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileContextResolveAfs.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileContextResolveAfs.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileContextResolveAfs.java index ca9de83..1fe82e9 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileContextResolveAfs.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileContextResolveAfs.java @@ -31,7 +31,9 @@ import org.junit.Test; * Tests resolution of AbstractFileSystems for a given path with symlinks. */ public class TestFileContextResolveAfs { - + static { + FileSystem.enableSymlinks(); + } private static String TEST_ROOT_DIR_LOCAL = System.getProperty("test.build.data","/tmp"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/26971e52/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestStat.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestStat.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestStat.java index 6265012..0e518e2 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestStat.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestStat.java @@ -32,7 +32,9 @@ import org.junit.BeforeClass; import org.junit.Test; public class TestStat extends FileSystemTestHelper { - + static { + FileSystem.enableSymlinks(); + } private static Stat stat; @BeforeClass http://git-wip-us.apache.org/repos/asf/hadoop/blob/26971e52/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 8d6a8fe..4ca6d57 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -1286,12 +1286,16 @@ public class DistributedFileSystem extends FileSystem { }.resolve(this, absF); } + @SuppressWarnings("deprecation") @Override public void createSymlink(final Path target, final Path link, final boolean createParent) throws AccessControlException, FileAlreadyExistsException, FileNotFoundException, ParentNotDirectoryException, UnsupportedFileSystemException, IOException { + if (!FileSystem.areSymlinksEnabled()) { + throw new UnsupportedOperationException("Symlinks not supported"); + } statistics.incrementWriteOps(1); final Path absF = fixRelativePart(link); new FileSystemLinkResolver<Void>() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/26971e52/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java index 293bcba..43171de 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java @@ -33,6 +33,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.XAttrSetFlag; import org.apache.hadoop.hdfs.protocol.HdfsConstantsClient; import org.apache.hadoop.hdfs.protocol.LocatedBlock; @@ -630,6 +631,9 @@ public class FSEditLogLoader { break; } case OP_SYMLINK: { + if (!FileSystem.areSymlinksEnabled()) { + throw new IOException("Symlinks not supported - please remove symlink before upgrading to this version of HDFS"); + } SymlinkOp symlinkOp = (SymlinkOp)op; inodeId = getAndUpdateLastInodeId(symlinkOp.inodeId, logVersion, lastInodeId); http://git-wip-us.apache.org/repos/asf/hadoop/blob/26971e52/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java index cce991f..906213e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java @@ -42,6 +42,7 @@ import org.apache.commons.logging.Log; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathIsNotDirectoryException; import org.apache.hadoop.fs.UnresolvedLinkException; @@ -723,6 +724,7 @@ public class FSImageFormat { * @param counter Counter to increment for namenode startup progress * @return an inode */ + @SuppressWarnings("deprecation") INode loadINode(final byte[] localName, boolean isSnapshotINode, DataInput in, Counter counter) throws IOException { final int imgVersion = getLayoutVersion(); @@ -836,6 +838,9 @@ public class FSImageFormat { return dir; } else if (numBlocks == -2) { //symlink + if (!FileSystem.areSymlinksEnabled()) { + throw new IOException("Symlinks not supported - please remove symlink before upgrading to this version of HDFS"); + } final String symlink = Text.readString(in); final PermissionStatus permissions = PermissionStatus.read(in); http://git-wip-us.apache.org/repos/asf/hadoop/blob/26971e52/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index e400644..3599fad 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -147,6 +147,7 @@ import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FileEncryptionInfo; import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FsServerDefaults; import org.apache.hadoop.fs.InvalidPathException; import org.apache.hadoop.fs.Options; @@ -2131,6 +2132,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, void createSymlink(String target, String link, PermissionStatus dirPerms, boolean createParent, boolean logRetryCache) throws IOException { + if (!FileSystem.areSymlinksEnabled()) { + throw new UnsupportedOperationException("Symlinks not supported"); + } waitForLoadingFSImage(); HdfsFileStatus auditStat = null; checkOperation(OperationCategory.WRITE); http://git-wip-us.apache.org/repos/asf/hadoop/blob/26971e52/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java index d3eaa6e..8658e47 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java @@ -769,6 +769,9 @@ public class MiniDFSCluster { try { ExitUtil.disableSystemExit(); + // Re-enable symlinks for tests, see HADOOP-10020 and HADOOP-10052 + FileSystem.enableSymlinks(); + synchronized (MiniDFSCluster.class) { instanceId = instanceCount++; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/26971e52/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java index 8e1f1ef..4e6c59a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java @@ -71,6 +71,10 @@ import org.mockito.Mockito; import com.google.common.collect.ImmutableList; public class TestINodeFile { + // Re-enable symlinks for tests, see HADOOP-10020 and HADOOP-10052 + static { + FileSystem.enableSymlinks(); + } public static final Log LOG = LogFactory.getLog(TestINodeFile.class); static final short BLOCKBITS = 48;