Author: jlowe Date: Wed Nov 13 23:23:00 2013 New Revision: 1541758 URL: http://svn.apache.org/r1541758 Log: HADOOP-10091. Job with a har archive as input fails on 0.23. Contributed by Jason Dere and Jason Lowe
Added: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/.part-0.crc (with props) hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/_SUCCESS (with props) hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/_index (with props) hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/_masterindex (with props) hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/part-0 (with props) Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/pom.xml hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1541758&r1=1541757&r2=1541758&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Wed Nov 13 23:23:00 2013 @@ -26,6 +26,9 @@ Release 0.23.10 - UNRELEASED HADOOP-9757. Har metadata cache can grow without limit (Cristina Abad via daryn) + HADOOP-10091. Job with a har archive as input fails on 0.23 (Jason Dere + and Jason Lowe via jlowe) + Release 0.23.9 - 2013-07-08 INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/pom.xml?rev=1541758&r1=1541757&r2=1541758&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/pom.xml (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/pom.xml Wed Nov 13 23:23:00 2013 @@ -486,6 +486,10 @@ <exclude>src/test/all-tests</exclude> <exclude>src/test/resources/kdc/ldif/users.ldif</exclude> <exclude>src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4.c</exclude> + <exclude>src/test/resources/test.har/_SUCCESS</exclude> + <exclude>src/test/resources/test.har/_index</exclude> + <exclude>src/test/resources/test.har/_masterindex</exclude> + <exclude>src/test/resources/test.har/part-0</exclude> </excludes> </configuration> </plugin> Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java?rev=1541758&r1=1541757&r2=1541758&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java Wed Nov 13 23:23:00 2013 @@ -1329,7 +1329,7 @@ public abstract class FileSystem extends return new ContentSummary(summary[0], summary[1], summary[2]); } - final private static PathFilter DEFAULT_FILTER = new PathFilter() { + final protected static PathFilter DEFAULT_FILTER = new PathFilter() { public boolean accept(Path file) { return true; } Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java?rev=1541758&r1=1541757&r2=1541758&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java Wed Nov 13 23:23:00 2013 @@ -697,6 +697,13 @@ public class HarFileSystem extends Filte throw new IOException("Har: delete not allowed"); } + @Override + public RemoteIterator<LocatedFileStatus> listLocatedStatus(Path f) + throws IOException { + // Use FileSystem's implementation + return listLocatedStatus(f, DEFAULT_FILTER); + } + /** * liststatus returns the children of a directory * after looking up the index files. Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java?rev=1541758&r1=1541757&r2=1541758&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java Wed Nov 13 23:23:00 2013 @@ -25,8 +25,12 @@ import static org.junit.Assert.assertFal import java.io.File; import java.io.IOException; import java.net.URI; +import java.util.HashSet; +import java.util.Set; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.LocatedFileStatus; +import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.util.Shell; import org.junit.After; @@ -221,6 +225,32 @@ public class TestHarFileSystemBasics { hfs.initialize(uri, new Configuration()); } + @Test + public void testListLocatedStatus() throws Exception { + String testHarPath = this.getClass().getResource("/test.har").getPath(); + URI uri = new URI("har://" + testHarPath); + HarFileSystem hfs = new HarFileSystem(localFileSystem); + hfs.initialize(uri, new Configuration()); + + // test.har has the following contents: + // dir1/1.txt + // dir1/2.txt + Set<String> expectedFileNames = new HashSet<String>(); + expectedFileNames.add("1.txt"); + expectedFileNames.add("2.txt"); + + // List contents of dir, and ensure we find all expected files + Path path = new Path("dir1"); + RemoteIterator<LocatedFileStatus> fileList = hfs.listLocatedStatus(path); + while (fileList.hasNext()) { + String fileName = fileList.next().getPath().getName(); + assertTrue(fileName + " not in expected files list", expectedFileNames.contains(fileName)); + expectedFileNames.remove(fileName); + } + assertEquals("Didn't find all of the expected file names: " + expectedFileNames, + 0, expectedFileNames.size()); + } + // ========== Negative: @Test Added: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/.part-0.crc URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/.part-0.crc?rev=1541758&view=auto ============================================================================== Binary file - no diff available. Propchange: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/.part-0.crc ------------------------------------------------------------------------------ svn:mime-type = application/octet-stream Added: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/_SUCCESS URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/_SUCCESS?rev=1541758&view=auto ============================================================================== (empty) Propchange: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/_SUCCESS ------------------------------------------------------------------------------ svn:executable = * Added: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/_index URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/_index?rev=1541758&view=auto ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/_index (added) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/_index Wed Nov 13 23:23:00 2013 @@ -0,0 +1,4 @@ +%2F dir 1380270822000+511+root+wheel 0 0 dir1 +%2Fdir1 dir 1380270441000+493+jdere+wheel 0 0 1.txt 2.txt +%2Fdir1%2F1.txt file part-0 0 0 1380270439000+420+jdere+wheel +%2Fdir1%2F2.txt file part-0 0 0 1380270441000+420+jdere+wheel Propchange: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/_index ------------------------------------------------------------------------------ svn:executable = * Added: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/_masterindex URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/_masterindex?rev=1541758&view=auto ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/_masterindex (added) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/_masterindex Wed Nov 13 23:23:00 2013 @@ -0,0 +1,2 @@ +3 +0 1210114968 0 232 Propchange: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/_masterindex ------------------------------------------------------------------------------ svn:executable = * Added: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/part-0 URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/part-0?rev=1541758&view=auto ============================================================================== (empty) Propchange: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/part-0 ------------------------------------------------------------------------------ svn:executable = *