Author: jlowe
Date: Wed Nov 13 23:23:00 2013
New Revision: 1541758
URL: http://svn.apache.org/r1541758
Log:
HADOOP-10091. Job with a har archive as input fails on 0.23. Contributed by
Jason Dere and Jason Lowe
Added:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/.part-0.crc
(with props)
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/_SUCCESS
(with props)
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/_index
(with props)
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/_masterindex
(with props)
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/part-0
(with props)
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/pom.xml
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1541758&r1=1541757&r2=1541758&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
Wed Nov 13 23:23:00 2013
@@ -26,6 +26,9 @@ Release 0.23.10 - UNRELEASED
HADOOP-9757. Har metadata cache can grow without limit (Cristina Abad via
daryn)
+ HADOOP-10091. Job with a har archive as input fails on 0.23 (Jason Dere
+ and Jason Lowe via jlowe)
+
Release 0.23.9 - 2013-07-08
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/pom.xml?rev=1541758&r1=1541757&r2=1541758&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/pom.xml
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/pom.xml
Wed Nov 13 23:23:00 2013
@@ -486,6 +486,10 @@
<exclude>src/test/all-tests</exclude>
<exclude>src/test/resources/kdc/ldif/users.ldif</exclude>
<exclude>src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4.c</exclude>
+ <exclude>src/test/resources/test.har/_SUCCESS</exclude>
+ <exclude>src/test/resources/test.har/_index</exclude>
+ <exclude>src/test/resources/test.har/_masterindex</exclude>
+ <exclude>src/test/resources/test.har/part-0</exclude>
</excludes>
</configuration>
</plugin>
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java?rev=1541758&r1=1541757&r2=1541758&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
Wed Nov 13 23:23:00 2013
@@ -1329,7 +1329,7 @@ public abstract class FileSystem extends
return new ContentSummary(summary[0], summary[1], summary[2]);
}
- final private static PathFilter DEFAULT_FILTER = new PathFilter() {
+ final protected static PathFilter DEFAULT_FILTER = new PathFilter() {
public boolean accept(Path file) {
return true;
}
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java?rev=1541758&r1=1541757&r2=1541758&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
Wed Nov 13 23:23:00 2013
@@ -697,6 +697,13 @@ public class HarFileSystem extends Filte
throw new IOException("Har: delete not allowed");
}
+ @Override
+ public RemoteIterator<LocatedFileStatus> listLocatedStatus(Path f)
+ throws IOException {
+ // Use FileSystem's implementation
+ return listLocatedStatus(f, DEFAULT_FILTER);
+ }
+
/**
* liststatus returns the children of a directory
* after looking up the index files.
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java?rev=1541758&r1=1541757&r2=1541758&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java
Wed Nov 13 23:23:00 2013
@@ -25,8 +25,12 @@ import static org.junit.Assert.assertFal
import java.io.File;
import java.io.IOException;
import java.net.URI;
+import java.util.HashSet;
+import java.util.Set;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.util.Shell;
import org.junit.After;
@@ -221,6 +225,32 @@ public class TestHarFileSystemBasics {
hfs.initialize(uri, new Configuration());
}
+ @Test
+ public void testListLocatedStatus() throws Exception {
+ String testHarPath = this.getClass().getResource("/test.har").getPath();
+ URI uri = new URI("har://" + testHarPath);
+ HarFileSystem hfs = new HarFileSystem(localFileSystem);
+ hfs.initialize(uri, new Configuration());
+
+ // test.har has the following contents:
+ // dir1/1.txt
+ // dir1/2.txt
+ Set<String> expectedFileNames = new HashSet<String>();
+ expectedFileNames.add("1.txt");
+ expectedFileNames.add("2.txt");
+
+ // List contents of dir, and ensure we find all expected files
+ Path path = new Path("dir1");
+ RemoteIterator<LocatedFileStatus> fileList = hfs.listLocatedStatus(path);
+ while (fileList.hasNext()) {
+ String fileName = fileList.next().getPath().getName();
+ assertTrue(fileName + " not in expected files list",
expectedFileNames.contains(fileName));
+ expectedFileNames.remove(fileName);
+ }
+ assertEquals("Didn't find all of the expected file names: " +
expectedFileNames,
+ 0, expectedFileNames.size());
+ }
+
// ========== Negative:
@Test
Added:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/.part-0.crc
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/.part-0.crc?rev=1541758&view=auto
==============================================================================
Binary file - no diff available.
Propchange:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/.part-0.crc
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/_SUCCESS
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/_SUCCESS?rev=1541758&view=auto
==============================================================================
(empty)
Propchange:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/_SUCCESS
------------------------------------------------------------------------------
svn:executable = *
Added:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/_index
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/_index?rev=1541758&view=auto
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/_index
(added)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/_index
Wed Nov 13 23:23:00 2013
@@ -0,0 +1,4 @@
+%2F dir 1380270822000+511+root+wheel 0 0 dir1
+%2Fdir1 dir 1380270441000+493+jdere+wheel 0 0 1.txt 2.txt
+%2Fdir1%2F1.txt file part-0 0 0 1380270439000+420+jdere+wheel
+%2Fdir1%2F2.txt file part-0 0 0 1380270441000+420+jdere+wheel
Propchange:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/_index
------------------------------------------------------------------------------
svn:executable = *
Added:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/_masterindex
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/_masterindex?rev=1541758&view=auto
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/_masterindex
(added)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/_masterindex
Wed Nov 13 23:23:00 2013
@@ -0,0 +1,2 @@
+3
+0 1210114968 0 232
Propchange:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/_masterindex
------------------------------------------------------------------------------
svn:executable = *
Added:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/part-0
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/part-0?rev=1541758&view=auto
==============================================================================
(empty)
Propchange:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/resources/test.har/part-0
------------------------------------------------------------------------------
svn:executable = *
