Author: eli Date: Mon Nov 21 07:28:49 2011 New Revision: 1204379 URL: http://svn.apache.org/viewvc?rev=1204379&view=rev Log: HADOOP-7783. svn merge -c 1204376 from trunk
Modified: hadoop/common/branches/branch-0.22/common/src/java/org/apache/hadoop/fs/AbstractFileSystem.java hadoop/common/branches/branch-0.22/common/src/java/org/apache/hadoop/fs/FileContext.java hadoop/common/branches/branch-0.22/common/src/java/org/apache/hadoop/fs/Path.java hadoop/common/branches/branch-0.22/common/src/test/core/org/apache/hadoop/fs/FileContextSymlinkBaseTest.java hadoop/common/branches/branch-0.22/common/src/test/core/org/apache/hadoop/fs/TestPath.java Modified: hadoop/common/branches/branch-0.22/common/src/java/org/apache/hadoop/fs/AbstractFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/common/src/java/org/apache/hadoop/fs/AbstractFileSystem.java?rev=1204379&r1=1204378&r2=1204379&view=diff ============================================================================== --- hadoop/common/branches/branch-0.22/common/src/java/org/apache/hadoop/fs/AbstractFileSystem.java (original) +++ hadoop/common/branches/branch-0.22/common/src/java/org/apache/hadoop/fs/AbstractFileSystem.java Mon Nov 21 07:28:49 2011 @@ -348,7 +348,7 @@ public abstract class AbstractFileSystem checkPath(p); String s = p.toUri().getPath(); if (!isValidName(s)) { - throw new InvalidPathException("Path part " + s + " from URI" + p + throw new InvalidPathException("Path part " + s + " from URI " + p + " is not a valid filename."); } return s; Modified: hadoop/common/branches/branch-0.22/common/src/java/org/apache/hadoop/fs/FileContext.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/common/src/java/org/apache/hadoop/fs/FileContext.java?rev=1204379&r1=1204378&r2=1204379&view=diff ============================================================================== --- hadoop/common/branches/branch-0.22/common/src/java/org/apache/hadoop/fs/FileContext.java (original) +++ hadoop/common/branches/branch-0.22/common/src/java/org/apache/hadoop/fs/FileContext.java Mon Nov 21 07:28:49 2011 @@ -1014,29 +1014,28 @@ public final class FileContext { * Return a fully qualified version of the given symlink target if it * has no scheme and authority. Partially and fully qualified paths * are returned unmodified. - * @param linkFS The AbstractFileSystem of link - * @param link The path of the symlink - * @param target The symlink's target + * @param pathFS The AbstractFileSystem of the path + * @param pathWithLink Path that contains the symlink + * @param target The symlink's absolute target * @return Fully qualified version of the target. */ - private Path qualifySymlinkTarget(final AbstractFileSystem linkFS, - Path link, Path target) { - /* NB: makeQualified uses link's scheme/authority, if specified, - * and the scheme/authority of linkFS, if not. If link does have - * a scheme and authority they should match those of linkFS since - * resolve updates the path and file system of a path that contains - * links each time a link is encountered. + private Path qualifySymlinkTarget(final AbstractFileSystem pathFS, + Path pathWithLink, Path target) { + /* NB: makeQualified uses the target's scheme and authority, if + * specified, and the scheme and authority of pathFS, if not. If + * the path does have a scheme and authority we assert they match + * those of pathFS since resolve updates the file system of a path + * that contains links each time a link is encountered. */ - final String linkScheme = link.toUri().getScheme(); - final String linkAuth = link.toUri().getAuthority(); - if (linkScheme != null && linkAuth != null) { - assert linkScheme.equals(linkFS.getUri().getScheme()); - assert linkAuth.equals(linkFS.getUri().getAuthority()); - } - final boolean justPath = (target.toUri().getScheme() == null && - target.toUri().getAuthority() == null); - return justPath ? target.makeQualified(linkFS.getUri(), link.getParent()) - : target; + final String scheme = target.toUri().getScheme(); + final String auth = target.toUri().getAuthority(); + if (scheme != null && auth != null) { + assert scheme.equals(pathFS.getUri().getScheme()); + assert auth.equals(pathFS.getUri().getAuthority()); + } + return (scheme == null && auth == null) + ? target.makeQualified(pathFS.getUri(), pathWithLink.getParent()) + : target; } /** @@ -1070,16 +1069,19 @@ public final class FileContext { } /** - * Returns the un-interpreted target of the given symbolic link. - * Transparently resolves all links up to the final path component. - * @param f + * Returns the target of the given symbolic link as it was specified + * when the link was created. Links in the path leading up to the + * final path component are resolved transparently. + * + * @param f the path to return the target of * @return The un-interpreted target of the symbolic link. * * @throws AccessControlException If access is denied * @throws FileNotFoundException If path <code>f</code> does not exist * @throws UnsupportedFileSystemException If file system for <code>f</code> is * not supported - * @throws IOException If an I/O error occurred + * @throws IOException If the given path does not refer to a symlink + * or an I/O error occurred */ public Path getLinkTarget(final Path f) throws AccessControlException, FileNotFoundException, UnsupportedFileSystemException, IOException { @@ -1199,7 +1201,7 @@ public final class FileContext { * getFsStatus, getFileStatus, exists, and listStatus. * * Symlink targets are stored as given to createSymlink, assuming the - * underlying file system is capable of storign a fully qualified URI. + * underlying file system is capable of storing a fully qualified URI. * Dangling symlinks are permitted. FileContext supports four types of * symlink targets, and resolves them as follows * <pre> Modified: hadoop/common/branches/branch-0.22/common/src/java/org/apache/hadoop/fs/Path.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/common/src/java/org/apache/hadoop/fs/Path.java?rev=1204379&r1=1204378&r2=1204379&view=diff ============================================================================== --- hadoop/common/branches/branch-0.22/common/src/java/org/apache/hadoop/fs/Path.java (original) +++ hadoop/common/branches/branch-0.22/common/src/java/org/apache/hadoop/fs/Path.java Mon Nov 21 07:28:49 2011 @@ -66,13 +66,14 @@ public class Path implements Comparable // Add a slash to parent's path so resolution is compatible with URI's URI parentUri = parent.uri; String parentPath = parentUri.getPath(); - if (!(parentPath.equals("/") || parentPath.equals(""))) + if (!(parentPath.equals("/") || parentPath.equals(""))) { try { parentUri = new URI(parentUri.getScheme(), parentUri.getAuthority(), parentUri.getPath()+"/", null, parentUri.getFragment()); } catch (URISyntaxException e) { throw new IllegalArgumentException(e); } + } URI resolved = parentUri.resolve(child.uri); initialize(resolved.getScheme(), resolved.getAuthority(), normalizePath(resolved.getPath()), resolved.getFragment()); @@ -211,7 +212,8 @@ public class Path implements Comparable * There is some ambiguity here. An absolute path is a slash * relative name without a scheme or an authority. * So either this method was incorrectly named or its - * implementation is incorrect. + * implementation is incorrect. This method returns true + * even if there is a scheme and authority. */ public boolean isAbsolute() { return isUriPathAbsolute(); @@ -305,19 +307,16 @@ public class Path implements Comparable return depth; } - /** * Returns a qualified path object. * * Deprecated - use {@link #makeQualified(URI, Path)} */ - @Deprecated public Path makeQualified(FileSystem fs) { return makeQualified(fs.getUri(), fs.getWorkingDirectory()); } - /** Returns a qualified path object. */ @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"}) public Path makeQualified(URI defaultUri, Path workingDir ) { Modified: hadoop/common/branches/branch-0.22/common/src/test/core/org/apache/hadoop/fs/FileContextSymlinkBaseTest.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/common/src/test/core/org/apache/hadoop/fs/FileContextSymlinkBaseTest.java?rev=1204379&r1=1204378&r2=1204379&view=diff ============================================================================== --- hadoop/common/branches/branch-0.22/common/src/test/core/org/apache/hadoop/fs/FileContextSymlinkBaseTest.java (original) +++ hadoop/common/branches/branch-0.22/common/src/test/core/org/apache/hadoop/fs/FileContextSymlinkBaseTest.java Mon Nov 21 07:28:49 2011 @@ -28,8 +28,9 @@ import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.FSDataOutputStream; import static org.apache.hadoop.fs.FileContextTestHelper.*; -import static org.junit.Assert.*; +import static org.junit.Assert.*; +import static org.junit.Assume.assumeTrue; import org.junit.Test; import org.junit.Before; import org.junit.After; @@ -238,6 +239,31 @@ public abstract class FileContextSymlink assertFalse(isDir(fc, linkToFile)); assertEquals(file.toUri().getPath(), fc.getLinkTarget(linkToFile).toString()); + // The local file system does not fully resolve the link + // when obtaining the file status + if (!"file".equals(getScheme())) { + assertEquals(fc.getFileStatus(file), fc.getFileStatus(linkToFile)); + assertEquals(fc.makeQualified(file), + fc.getFileStatus(linkToFile).getPath()); + assertEquals(fc.makeQualified(linkToFile), + fc.getFileLinkStatus(linkToFile).getPath()); + } + } + + @Test + /** Stat a relative link to a file */ + public void testStatRelLinkToFile() throws IOException { + assumeTrue(!"file".equals(getScheme())); + Path baseDir = new Path(testBaseDir1()); + Path file = new Path(testBaseDir1(), "file"); + Path linkToFile = new Path(testBaseDir1(), "linkToFile"); + createAndWriteFile(file); + fc.createSymlink(new Path("file"), linkToFile, false); + assertEquals(fc.getFileStatus(file), fc.getFileStatus(linkToFile)); + assertEquals(fc.makeQualified(file), + fc.getFileStatus(linkToFile).getPath()); + assertEquals(fc.makeQualified(linkToFile), + fc.getFileLinkStatus(linkToFile).getPath()); } @Test @@ -474,18 +500,15 @@ public abstract class FileContextSymlink * creating using a partially qualified path is file system specific. */ public void testCreateLinkUsingPartQualPath1() throws IOException { + // Partially qualified paths are covered for local file systems + // in the previous test. + assumeTrue(!"file".equals(getScheme())); Path schemeAuth = new Path(testURI().toString()); Path fileWoHost = new Path(getScheme()+"://"+testBaseDir1()+"/file"); Path link = new Path(testBaseDir1()+"/linkToFile"); Path linkQual = new Path(schemeAuth, testBaseDir1()+"/linkToFile"); - - // Partially qualified paths are covered for local file systems - // in the previous test. - if ("file".equals(getScheme())) { - return; - } FileContext localFc = FileContext.getLocalFSFileContext(); - + fc.createSymlink(fileWoHost, link, false); // Partially qualified path is stored assertEquals(fileWoHost, fc.getLinkTarget(linkQual)); @@ -748,7 +771,7 @@ public abstract class FileContextSymlink } @Test - /** Test create symlink to ../foo */ + /** Test create symlink to ../file */ public void testCreateLinkToDotDotPrefix() throws IOException { Path file = new Path(testBaseDir1(), "file"); Path dir = new Path(testBaseDir1(), "test"); @@ -1205,24 +1228,30 @@ public abstract class FileContextSymlink } @Test - /** Operate on a file using a path with an intermediate symlink */ - public void testAccessFileViaSymlink() throws IOException { + /** + * Create, write, read, append, rename, get the block locations, + * checksums, and delete a file using a path with a symlink as an + * intermediate path component where the link target was specified + * using an absolute path. Rename is covered in more depth below. + */ + public void testAccessFileViaInterSymlinkAbsTarget() throws IOException { Path baseDir = new Path(testBaseDir1()); + Path file = new Path(testBaseDir1(), "file"); Path fileNew = new Path(baseDir, "fileNew"); Path linkToDir = new Path(testBaseDir2(), "linkToDir"); Path fileViaLink = new Path(linkToDir, "file"); Path fileNewViaLink = new Path(linkToDir, "fileNew"); fc.createSymlink(baseDir, linkToDir, false); - // Create, write, read, append, rename, get block locations and - // checksums, and delete a file using a path that contains a - // symlink as an intermediate path component. Rename is covered - // in more depth below. createAndWriteFile(fileViaLink); assertTrue(exists(fc, fileViaLink)); assertTrue(isFile(fc, fileViaLink)); assertFalse(isDir(fc, fileViaLink)); assertFalse(fc.getFileLinkStatus(fileViaLink).isSymlink()); assertFalse(isDir(fc, fileViaLink)); + assertEquals(fc.getFileStatus(file), + fc.getFileLinkStatus(file)); + assertEquals(fc.getFileStatus(fileViaLink), + fc.getFileLinkStatus(fileViaLink)); readFile(fileViaLink); appendToFile(fileViaLink); fc.rename(fileViaLink, fileNewViaLink); @@ -1238,6 +1267,58 @@ public abstract class FileContextSymlink } @Test + /** + * Operate on a file using a path with an intermediate symlink where + * the link target was specified as a fully qualified path. + */ + public void testAccessFileViaInterSymlinkQualTarget() throws IOException { + Path baseDir = new Path(testBaseDir1()); + Path file = new Path(testBaseDir1(), "file"); + Path fileNew = new Path(baseDir, "fileNew"); + Path linkToDir = new Path(testBaseDir2(), "linkToDir"); + Path fileViaLink = new Path(linkToDir, "file"); + Path fileNewViaLink = new Path(linkToDir, "fileNew"); + fc.createSymlink(fc.makeQualified(baseDir), linkToDir, false); + createAndWriteFile(fileViaLink); + assertEquals(fc.getFileStatus(file), + fc.getFileLinkStatus(file)); + assertEquals(fc.getFileStatus(fileViaLink), + fc.getFileLinkStatus(fileViaLink)); + readFile(fileViaLink); + } + + @Test + /** + * Operate on a file using a path with an intermediate symlink where + * the link target was specified as a relative path. + */ + public void testAccessFileViaInterSymlinkRelTarget() throws IOException { + assumeTrue(!"file".equals(getScheme())); + Path baseDir = new Path(testBaseDir1()); + Path dir = new Path(testBaseDir1(), "dir"); + Path file = new Path(dir, "file"); + Path linkToDir = new Path(testBaseDir1(), "linkToDir"); + Path fileViaLink = new Path(linkToDir, "file"); + + fc.mkdir(dir, FileContext.DEFAULT_PERM, false); + fc.createSymlink(new Path("dir"), linkToDir, false); + createAndWriteFile(fileViaLink); + // Note that getFileStatus returns fully qualified paths even + // when called on an absolute path. + assertEquals(fc.makeQualified(file), + fc.getFileStatus(file).getPath()); + // In each case getFileLinkStatus returns the same FileStatus + // as getFileStatus since we're not calling it on a link and + // FileStatus objects are compared by Path. + assertEquals(fc.getFileStatus(file), + fc.getFileLinkStatus(file)); + assertEquals(fc.getFileStatus(fileViaLink), + fc.getFileLinkStatus(fileViaLink)); + assertEquals(fc.getFileStatus(fileViaLink), + fc.getFileLinkStatus(file)); + } + + @Test /** Test create, list, and delete a directory through a symlink */ public void testAccessDirViaSymlink() throws IOException { Path baseDir = new Path(testBaseDir1()); @@ -1272,4 +1353,4 @@ public abstract class FileContextSymlink assertEquals(2, fc.getFileStatus(file).getModificationTime()); } } -} \ No newline at end of file +} Modified: hadoop/common/branches/branch-0.22/common/src/test/core/org/apache/hadoop/fs/TestPath.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/common/src/test/core/org/apache/hadoop/fs/TestPath.java?rev=1204379&r1=1204378&r2=1204379&view=diff ============================================================================== --- hadoop/common/branches/branch-0.22/common/src/test/core/org/apache/hadoop/fs/TestPath.java (original) +++ hadoop/common/branches/branch-0.22/common/src/test/core/org/apache/hadoop/fs/TestPath.java Mon Nov 21 07:28:49 2011 @@ -95,6 +95,7 @@ public class TestPath extends TestCase { assertEquals(new Path("/foo"), new Path("/foo/bar").getParent()); assertEquals(new Path("foo"), new Path("foo/bar").getParent()); assertEquals(new Path("/"), new Path("/foo").getParent()); + assertEquals(null, new Path("/").getParent()); if (Path.WINDOWS) { assertEquals(new Path("c:/"), new Path("c:/foo").getParent()); } @@ -159,6 +160,13 @@ public class TestPath extends TestCase { assertEquals(new Path("foo/bar/baz","../../..").toString(), ""); assertEquals(new Path("foo/bar/baz","../../../../..").toString(), "../.."); } + + /** Test Path objects created from other Path objects */ + public void testChildParentResolution() throws URISyntaxException, IOException { + Path parent = new Path("foo1://bar1/baz1"); + Path child = new Path("foo2://bar2/baz2"); + assertEquals(child, new Path(parent, child)); + } public void testScheme() throws java.io.IOException { assertEquals("foo:/bar", new Path("foo:/","/bar").toString());