Repository: hadoop Updated Branches: refs/heads/branch-2 a96696bdc -> 0a582c77e
HADOOP-11012. hadoop fs -text of zero-length file causes EOFException. Contributed by Eric Payne (cherry picked from commit 01e8f056d9b7245193e6050f9830ca058db02a6e) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0a582c77 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0a582c77 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0a582c77 Branch: refs/heads/branch-2 Commit: 0a582c77efc761bca9c388841010dd67e49b8a26 Parents: a96696b Author: Jason Lowe <jl...@apache.org> Authored: Tue Sep 2 18:25:03 2014 +0000 Committer: Jason Lowe <jl...@apache.org> Committed: Tue Sep 2 18:25:03 2014 +0000 ---------------------------------------------------------------------- hadoop-common-project/hadoop-common/CHANGES.txt | 3 + .../org/apache/hadoop/fs/shell/Display.java | 12 ++- .../apache/hadoop/fs/shell/TestTextCommand.java | 77 +++++++++++++++----- 3 files changed, 72 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a582c77/hadoop-common-project/hadoop-common/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 6bd3521..ecef57d 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -397,6 +397,9 @@ Release 2.6.0 - UNRELEASED HADOOP-11036. Add build directory to .gitignore (Tsuyoshi OZAWA via aw) + HADOOP-11012. hadoop fs -text of zero-length file causes EOFException + (Eric Payne via jlowe) + Release 2.5.1 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a582c77/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java index aa2dc49..3d607e4 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java @@ -18,6 +18,7 @@ package org.apache.hadoop.fs.shell; import java.io.ByteArrayOutputStream; +import java.io.EOFException; import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -126,8 +127,17 @@ class Display extends FsCommand { protected InputStream getInputStream(PathData item) throws IOException { FSDataInputStream i = (FSDataInputStream)super.getInputStream(item); + // Handle 0 and 1-byte files + short leadBytes; + try { + leadBytes = i.readShort(); + } catch (EOFException e) { + i.seek(0); + return i; + } + // Check type of stream first - switch(i.readShort()) { + switch(leadBytes) { case 0x1f8b: { // RFC 1952 // Must be gzip i.seek(0); http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a582c77/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java index 0c8a6ac..70a2f03 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java @@ -42,29 +42,14 @@ public class TestTextCommand { System.getProperty("test.build.data", "build/test/data/") + "/testText"; private static final String AVRO_FILENAME = new Path(TEST_ROOT_DIR, "weather.avro").toUri().getPath(); + private static final String TEXT_FILENAME = + new Path(TEST_ROOT_DIR, "testtextfile.txt").toUri().getPath(); /** * Tests whether binary Avro data files are displayed correctly. */ @Test (timeout = 30000) public void testDisplayForAvroFiles() throws Exception { - // Create a small Avro data file on the local file system. - createAvroFile(generateWeatherAvroBinaryData()); - - // Prepare and call the Text command's protected getInputStream method - // using reflection. - Configuration conf = new Configuration(); - URI localPath = new URI(AVRO_FILENAME); - PathData pathData = new PathData(localPath, conf); - Display.Text text = new Display.Text(); - text.setConf(conf); - Method method = text.getClass().getDeclaredMethod( - "getInputStream", PathData.class); - method.setAccessible(true); - InputStream stream = (InputStream) method.invoke(text, pathData); - String output = inputStreamToString(stream); - - // Check the output. String expectedOutput = "{\"station\":\"011990-99999\",\"time\":-619524000000,\"temp\":0}" + System.getProperty("line.separator") + @@ -77,18 +62,72 @@ public class TestTextCommand { "{\"station\":\"012650-99999\",\"time\":-655509600000,\"temp\":78}" + System.getProperty("line.separator"); + String output = readUsingTextCommand(AVRO_FILENAME, + generateWeatherAvroBinaryData()); assertEquals(expectedOutput, output); } + /** + * Tests that a zero-length file is displayed correctly. + */ + @Test (timeout = 30000) + public void testEmptyTextFil() throws Exception { + byte[] emptyContents = { }; + String output = readUsingTextCommand(TEXT_FILENAME, emptyContents); + assertTrue("".equals(output)); + } + + /** + * Tests that a one-byte file is displayed correctly. + */ + @Test (timeout = 30000) + public void testOneByteTextFil() throws Exception { + byte[] oneByteContents = { 'x' }; + String output = readUsingTextCommand(TEXT_FILENAME, oneByteContents); + assertTrue(new String(oneByteContents).equals(output)); + } + + /** + * Tests that a one-byte file is displayed correctly. + */ + @Test (timeout = 30000) + public void testTwoByteTextFil() throws Exception { + byte[] twoByteContents = { 'x', 'y' }; + String output = readUsingTextCommand(TEXT_FILENAME, twoByteContents); + assertTrue(new String(twoByteContents).equals(output)); + } + + // Create a file on the local file system and read it using + // the Display.Text class. + private String readUsingTextCommand(String fileName, byte[] fileContents) + throws Exception { + createFile(fileName, fileContents); + + // Prepare and call the Text command's protected getInputStream method + // using reflection. + Configuration conf = new Configuration(); + URI localPath = new URI(fileName); + PathData pathData = new PathData(localPath, conf); + Display.Text text = new Display.Text() { + @Override + public InputStream getInputStream(PathData item) throws IOException { + return super.getInputStream(item); + } + }; + text.setConf(conf); + InputStream stream = (InputStream) text.getInputStream(pathData); + return inputStreamToString(stream); + } + private String inputStreamToString(InputStream stream) throws IOException { StringWriter writer = new StringWriter(); IOUtils.copy(stream, writer); return writer.toString(); } - private void createAvroFile(byte[] contents) throws IOException { + private void createFile(String fileName, byte[] contents) throws IOException { (new File(TEST_ROOT_DIR)).mkdir(); - File file = new File(AVRO_FILENAME); + File file = new File(fileName); file.createNewFile(); FileOutputStream stream = new FileOutputStream(file); stream.write(contents);
