Author: rding Date: Thu Aug 5 17:33:11 2010 New Revision: 982704 URL: http://svn.apache.org/viewvc?rev=982704&view=rev Log: PIG-1533: Compression codec should be a per-store property
Modified: hadoop/pig/trunk/src/org/apache/pig/builtin/PigStorage.java hadoop/pig/trunk/test/org/apache/pig/test/TestBZip.java Modified: hadoop/pig/trunk/src/org/apache/pig/builtin/PigStorage.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/builtin/PigStorage.java?rev=982704&r1=982703&r2=982704&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/builtin/PigStorage.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/builtin/PigStorage.java Thu Aug 5 17:33:11 2010 @@ -238,6 +238,8 @@ LoadPushDown { } else if (location.endsWith(".gz")) { FileOutputFormat.setCompressOutput(job, true); FileOutputFormat.setOutputCompressorClass(job, GzipCodec.class); + } else { + FileOutputFormat.setCompressOutput(job, false); } } Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestBZip.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestBZip.java?rev=982704&r1=982703&r2=982704&view=diff ============================================================================== --- hadoop/pig/trunk/test/org/apache/pig/test/TestBZip.java (original) +++ hadoop/pig/trunk/test/org/apache/pig/test/TestBZip.java Thu Aug 5 17:33:11 2010 @@ -17,6 +17,7 @@ */ package org.apache.pig.test; +import static org.junit.Assert.*; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotSame; @@ -353,4 +354,31 @@ public class TestBZip { assertEquals(expectedCount, result); } + + @Test + public void testBzipStoreInMultiQuery() throws Exception { + String[] inputData = new String[] { + "1\t2\r3\t4" + }; + + String inputFileName = "input.txt"; + Util.createInputFile(cluster, inputFileName, inputData); + + PigServer pig = new PigServer(ExecType.MAPREDUCE, cluster + .getProperties()); + + pig.setBatchOn(); + pig.registerQuery("a = load '" + inputFileName + "';"); + pig.registerQuery("store a into 'output.bz2';"); + pig.registerQuery("store a into 'output';"); + pig.executeBatch(); + + FileSystem fs = FileSystem.get(ConfigurationUtil.toConfiguration( + pig.getPigContext().getProperties())); + FileStatus stat = fs.getFileStatus(new Path("output/part-m-00000")); + assertTrue(stat.getLen() > 0); + + stat = fs.getFileStatus(new Path("output.bz2/part-m-00000.bz2")); + assertTrue(stat.getLen() > 0); + } }