Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTextOutputFormat.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTextOutputFormat.java?rev=1462698&r1=1462697&r2=1462698&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTextOutputFormat.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTextOutputFormat.java Sat Mar 30 03:50:03 2013 @@ -44,7 +44,6 @@ public class TestTextOutputFormat extend "data"), FileOutputCommitter.TEMP_DIR_NAME), "_" + attempt); - @SuppressWarnings("unchecked") public void testFormat() throws Exception { JobConf job = new JobConf(); job.set(JobContext.TASK_ATTEMPT_ID, attempt); @@ -59,8 +58,8 @@ public class TestTextOutputFormat extend // A reporter that does nothing Reporter reporter = Reporter.NULL; - TextOutputFormat theOutputFormat = new TextOutputFormat(); - RecordWriter theRecordWriter = + TextOutputFormat<Object,Object> theOutputFormat = new TextOutputFormat<Object,Object>(); + RecordWriter<Object,Object> theRecordWriter = theOutputFormat.getRecordWriter(localFs, job, file, reporter); Text key1 = new Text("key1"); @@ -95,7 +94,6 @@ public class TestTextOutputFormat extend } - @SuppressWarnings("unchecked") public void testFormatWithCustomSeparator() throws Exception { JobConf job = new JobConf(); String separator = "\u0001"; @@ -112,8 +110,8 @@ public class TestTextOutputFormat extend // A reporter that does nothing Reporter reporter = Reporter.NULL; - TextOutputFormat theOutputFormat = new TextOutputFormat(); - RecordWriter theRecordWriter = + TextOutputFormat<Object,Object> theOutputFormat = new TextOutputFormat<Object,Object>(); + RecordWriter<Object,Object> theRecordWriter = theOutputFormat.getRecordWriter(localFs, job, file, reporter); Text key1 = new Text("key1"); @@ -147,7 +145,61 @@ public class TestTextOutputFormat extend assertEquals(output, expectedOutput.toString()); } - + /** + * test compressed file + * @throws IOException + */ + public void testCompress() throws IOException{ + JobConf job = new JobConf(); + String separator = "\u0001"; + job.set("mapreduce.output.textoutputformat.separator", separator); + job.set(JobContext.TASK_ATTEMPT_ID, attempt); + job.set(org.apache.hadoop.mapreduce.lib.output.FileOutputFormat.COMPRESS,"true"); + + FileOutputFormat.setOutputPath(job, workDir.getParent().getParent()); + FileOutputFormat.setWorkOutputPath(job, workDir); + FileSystem fs = workDir.getFileSystem(job); + if (!fs.mkdirs(workDir)) { + fail("Failed to create output directory"); + } + String file = "test.txt"; + + // A reporter that does nothing + Reporter reporter = Reporter.NULL; + + TextOutputFormat<Object,Object> theOutputFormat = new TextOutputFormat<Object,Object>(); + RecordWriter<Object,Object> theRecordWriter = + theOutputFormat.getRecordWriter(localFs, job, file, reporter); + Text key1 = new Text("key1"); + Text key2 = new Text("key2"); + Text val1 = new Text("val1"); + Text val2 = new Text("val2"); + NullWritable nullWritable = NullWritable.get(); + + try { + theRecordWriter.write(key1, val1); + theRecordWriter.write(null, nullWritable); + theRecordWriter.write(null, val1); + theRecordWriter.write(nullWritable, val2); + theRecordWriter.write(key2, nullWritable); + theRecordWriter.write(key1, null); + theRecordWriter.write(null, null); + theRecordWriter.write(key2, val2); + + } finally { + theRecordWriter.close(reporter); + } + File expectedFile = new File(new Path(workDir, file).toString()); + StringBuffer expectedOutput = new StringBuffer(); + expectedOutput.append(key1).append(separator).append(val1).append("\n"); + expectedOutput.append(val1).append("\n"); + expectedOutput.append(val2).append("\n"); + expectedOutput.append(key2).append("\n"); + expectedOutput.append(key1).append("\n"); + expectedOutput.append(key2).append(separator).append(val2).append("\n"); + String output = UtilsForTests.slurp(expectedFile); + assertEquals(output, expectedOutput.toString()); + } public static void main(String[] args) throws Exception { new TestTextOutputFormat().testFormat(); }
